From 15d84b5bd54226693aaa7e3ae26f06a837906aa1 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Thu, 18 Mar 2021 19:58:23 +0100 Subject: [PATCH 01/31] Update OS for enterprise ship:docker --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4dfbb9dbc6..12e8e4b692 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,14 @@ RUN tar -cjf public.tar.bz2 public && rm -rf public FROM ruby:2.5.8-slim LABEL maintainer Travis CI GmbH + +RUN ( \ + apt-get update ; \ + # update to deb 10.8 + apt-get upgrade -y ; \ + rm -rf /var/lib/apt/lists/* \ +) + WORKDIR /app ENV TRAVIS_BUILD_DUMP_BACKTRACE true From 9e5a7679e0aa5d34c82a559b158c5f152decfe96 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Tue, 5 Jan 2021 15:12:58 -0500 Subject: [PATCH 02/31] Update ghc.json --- public/version-aliases/ghc.json | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index e6951afb97..b33fc86cf1 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -115,9 +115,9 @@ "7.10.2": "7.10.2", "7.10.3": "7.10.3", "8.0.1": "8.0.1", - "8": "8.10.2", - "8.x": "8.10.2", - "8.x.x": "8.10.2", + "8": "8.10.3", + "8.x": "8.10.3", + "8.x.x": "8.10.3", "8.0.x": "8.0.2", "8.0": "8.0.2", "8.0.2": "8.0.2", @@ -153,9 +153,12 @@ "8.8.3": "8.8.3", "8.8.4": "8.8.4", "8.10.1-alpha1": "8.10.1-alpha1", - "8.10": "8.10.2", + "8.10": "8.10.3", "8.10.1-alpha2": "8.10.1-alpha2", "8.10.1": "8.10.1", - "8.10.x": "8.10.2", - "8.10.2": "8.10.2" -} \ No newline at end of file + "8.10.x": "8.10.3", + "8.10.2": "8.10.2", + "8.10.3": "8.10.3", + "9.0.1-alpha1": "9.0.1-alpha1", + "9.0": "9.0.1-alpha1" +} From eec8f721bb25c08b566506de755edc8623f96969 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Wed, 24 Mar 2021 14:31:46 +0100 Subject: [PATCH 03/31] Update ghc ship:docker --- public/version-aliases/ghc.json | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index b33fc86cf1..2e5fcf9b0e 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -115,9 +115,9 @@ "7.10.2": "7.10.2", "7.10.3": "7.10.3", "8.0.1": "8.0.1", - "8": "8.10.3", - "8.x": "8.10.3", - "8.x.x": "8.10.3", + "8": "8.10.4", + "8.x": "8.10.4", + "8.x.x": "8.10.4", "8.0.x": "8.0.2", "8.0": "8.0.2", "8.0.2": "8.0.2", @@ -153,12 +153,18 @@ "8.8.3": "8.8.3", "8.8.4": "8.8.4", "8.10.1-alpha1": "8.10.1-alpha1", - "8.10": "8.10.3", + "8.10": "8.10.4", "8.10.1-alpha2": "8.10.1-alpha2", "8.10.1": "8.10.1", - "8.10.x": "8.10.3", + "8.10.x": "8.10.4", "8.10.2": "8.10.2", "8.10.3": "8.10.3", + "8.10.4": "8.10.4", "9.0.1-alpha1": "9.0.1-alpha1", - "9.0": "9.0.1-alpha1" -} + "9.0": "9.0.1", + "9.0.1": "9.0.1", + "9": "9.0.1", + "9.x": "9.0.1", + "9.x.x": "9.0.1", + "9.0.x": "9.0.1" +} \ No newline at end of file From c6fef74e731a4deae807f4a09145943b86554ba0 Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Wed, 21 Apr 2021 14:56:30 +0200 Subject: [PATCH 04/31] Ext gem update (#1986) * security updates * build fix --- Gemfile | 2 +- Gemfile.lock | 16 ++++++++-------- public/version-aliases/ghc.json | 4 +++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index eacf922bc2..eb2fb481d1 100644 --- a/Gemfile +++ b/Gemfile @@ -10,7 +10,7 @@ def gh(slug) "https://github.com/#{slug}" end -gem 'activesupport', '~> 4.0' +gem 'activesupport', '~> 5' gem 'addressable', '~> 2.3' gem 'codeclimate-test-reporter', require: false, group: %i[development test] gem 'coder' diff --git a/Gemfile.lock b/Gemfile.lock index 633e22e2df..36e5d1b310 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,10 +37,10 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (4.2.11.1) - i18n (~> 0.7) + activesupport (5.2.5) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) addressable (2.5.2) public_suffix (>= 2.0.2, < 4.0) @@ -52,7 +52,7 @@ GEM simplecov (<= 0.13) coder (0.4.0) coderay (1.1.2) - concurrent-ruby (1.1.6) + concurrent-ruby (1.1.8) connection_pool (2.2.2) diff-lcs (1.3) docile (1.1.5) @@ -73,7 +73,7 @@ GEM hashr (2.0.1) highline (1.7.10) hitimes (1.3.0) - i18n (0.9.5) + i18n (1.8.10) concurrent-ruby (~> 1.0) jaro_winkler (1.5.1) json (2.3.1) @@ -91,7 +91,7 @@ GEM avl_tree (~> 1.1.2) hitimes (~> 1.1) minitar (0.6.1) - minitest (5.14.0) + minitest (5.14.4) mocha (1.7.0) metaclass (~> 0.0.1) msgpack (1.2.4) @@ -200,7 +200,7 @@ GEM hashr (~> 2.0) typhoeus (0.8.0) ethon (>= 0.8.0) - tzinfo (1.2.7) + tzinfo (1.2.9) thread_safe (~> 0.1) unicode-display_width (1.4.0) websocket (1.2.8) @@ -209,7 +209,7 @@ PLATFORMS ruby DEPENDENCIES - activesupport (~> 4.0) + activesupport (~> 5) addressable (~> 2.3) codeclimate-test-reporter coder diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 2e5fcf9b0e..3ae4f8b4a6 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,5 +166,7 @@ "9": "9.0.1", "9.x": "9.0.1", "9.x.x": "9.0.1", - "9.0.x": "9.0.1" + "9.0.x": "9.0.1", + "9.2.1-alpha1": "9.2.1-alpha1", + "9.2": "9.2.1-alpha1" } \ No newline at end of file From e91faff3beba582bfb2e2393906f6867ceeceb00 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 10 Mar 2021 09:53:58 +0100 Subject: [PATCH 05/31] PHP8 serialization fix Fix for PHP8 issue with build script on custom workers --- lib/travis/build/script/php.rb | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/travis/build/script/php.rb b/lib/travis/build/script/php.rb index 707011aa6c..2184fd3d34 100644 --- a/lib/travis/build/script/php.rb +++ b/lib/travis/build/script/php.rb @@ -221,23 +221,23 @@ def php_5_3_or_older? def overwrite_pearrc(version) pear_config = %q( [ - 'preferred_state' => "stable", - 'temp_dir' => "/tmp/pear/install", - 'download_dir' => "/tmp/pear/install", - 'bin_dir' => "/home/travis/.phpenv/versions/__VERSION__/bin", - 'php_dir' => "/home/travis/.phpenv/versions/__VERSION__/share/pear", - 'doc_dir' => "/home/travis/.phpenv/versions/__VERSION__/docs", - 'data_dir' => "/home/travis/.phpenv/versions/__VERSION__/data", - 'cfg_dir' => "/home/travis/.phpenv/versions/__VERSION__/cfg", - 'www_dir' => "/home/travis/.phpenv/versions/__VERSION__/www", - 'man_dir' => "/home/travis/.phpenv/versions/__VERSION__/man", - 'test_dir' => "/home/travis/.phpenv/versions/__VERSION__/tests", - '__channels' => [ - '__uri' => [], - 'doc.php.net' => [], - 'pecl.php.net' => [] + "preferred_state" => "stable", + "temp_dir" => "/tmp/pear/install", + "download_dir" => "/tmp/pear/install", + "bin_dir" => "/home/travis/.phpenv/versions/__VERSION__/bin", + "php_dir" => "/home/travis/.phpenv/versions/__VERSION__/share/pear", + "doc_dir" => "/home/travis/.phpenv/versions/__VERSION__/docs", + "data_dir" => "/home/travis/.phpenv/versions/__VERSION__/data", + "cfg_dir" => "/home/travis/.phpenv/versions/__VERSION__/cfg", + "www_dir" => "/home/travis/.phpenv/versions/__VERSION__/www", + "man_dir" => "/home/travis/.phpenv/versions/__VERSION__/man", + "test_dir" => "/home/travis/.phpenv/versions/__VERSION__/tests", + "__channels" => [ + "__uri" => [], + "doc.php.net" => [], + "pecl.php.net" => [] ], - 'auto_discover' => 1 + "auto_discover" => 1 ] ).gsub("__VERSION__", version) From dba049c33015e209a1a5c34679a24d36f42e69fb Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Fri, 23 Apr 2021 11:16:15 +0200 Subject: [PATCH 06/31] ghc update --- public/version-aliases/ghc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 3ae4f8b4a6..47e2b09e74 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -168,5 +168,6 @@ "9.x.x": "9.0.1", "9.0.x": "9.0.1", "9.2.1-alpha1": "9.2.1-alpha1", - "9.2": "9.2.1-alpha1" + "9.2": "9.2.1-alpha2", + "9.2.1-alpha2": "9.2.1-alpha2" } \ No newline at end of file From 92424327776f030bd28ef6707f46ea4902181f82 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Tue, 6 Jul 2021 16:05:29 +0200 Subject: [PATCH 07/31] docker fix --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 21e366c277..2f7cb7f74e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ env: addons: snaps: - name: docker - channel: latest/beta + channel: latest/stable stages: - test From eef0d9649fbc928f7efa5e8d66eab45e1a470dc5 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 7 Jul 2021 09:23:13 +0200 Subject: [PATCH 08/31] ghc update --- public/version-aliases/ghc.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 47e2b09e74..ea4a3a9cc7 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -115,9 +115,9 @@ "7.10.2": "7.10.2", "7.10.3": "7.10.3", "8.0.1": "8.0.1", - "8": "8.10.4", - "8.x": "8.10.4", - "8.x.x": "8.10.4", + "8": "8.10.5", + "8.x": "8.10.5", + "8.x.x": "8.10.5", "8.0.x": "8.0.2", "8.0": "8.0.2", "8.0.2": "8.0.2", @@ -153,13 +153,14 @@ "8.8.3": "8.8.3", "8.8.4": "8.8.4", "8.10.1-alpha1": "8.10.1-alpha1", - "8.10": "8.10.4", + "8.10": "8.10.5", "8.10.1-alpha2": "8.10.1-alpha2", "8.10.1": "8.10.1", - "8.10.x": "8.10.4", + "8.10.x": "8.10.5", "8.10.2": "8.10.2", "8.10.3": "8.10.3", "8.10.4": "8.10.4", + "8.10.5": "8.10.5", "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.1", "9.0.1": "9.0.1", From 8ba7820832a796fe12c592b70e1c36899a9add3a Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Thu, 29 Jul 2021 12:51:47 +0200 Subject: [PATCH 09/31] security updates [ship:docker] (#1998) --- Gemfile | 2 +- Gemfile.lock | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index eb2fb481d1..b64427ece3 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,7 @@ gem 'minitar' gem 'mocha', require: false, group: %i[development test] gem 'parallel_tests', require: false, group: %i[development test] gem 'pry', require: false, group: %i[development test] -gem 'puma' +gem 'puma', '~> 4' gem 'rack', '>= 2.1.4' gem 'rack-ssl', '~> 1.4' gem 'rack-test' diff --git a/Gemfile.lock b/Gemfile.lock index 36e5d1b310..baf7ea5aa8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -42,8 +42,8 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - addressable (2.5.2) - public_suffix (>= 2.0.2, < 4.0) + addressable (2.8.0) + public_suffix (>= 2.0.2, < 5.0) ast (2.4.0) atomic (1.1.101) avl_tree (1.1.3) @@ -102,6 +102,7 @@ GEM net-http-persistent (3.0.0) connection_pool (~> 2.2) net-http-pipeline (1.0.1) + nio4r (2.5.7) octokit (4.18.0) faraday (>= 0.9) sawyer (~> 0.8.0, >= 0.5.3) @@ -115,8 +116,9 @@ GEM pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) - public_suffix (3.0.3) - puma (3.12.6) + public_suffix (4.0.6) + puma (4.3.8) + nio4r (~> 2.0) pusher-client (0.6.2) json websocket (~> 1.0) @@ -225,7 +227,7 @@ DEPENDENCIES octokit (~> 4.18) parallel_tests pry - puma + puma (~> 4) rack (>= 2.1.4) rack-ssl (~> 1.4) rack-test From f94260f0673a156da6c4e9bcc8038a976c52b456 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Thu, 21 Oct 2021 11:56:18 +0200 Subject: [PATCH 10/31] update to use latest docker --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f7cb7f74e..81df6cd08b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: ruby dist: xenial rvm: 2.5.8 +group: edge services: - redis @@ -16,11 +17,6 @@ env: - INTEGRATION_SPECS=0 - INTEGRATION_SPECS=1 -addons: - snaps: - - name: docker - channel: latest/stable - stages: - test - ':ship: it to quay.io' From f3daad7afa95db3005464db42331372cda28d607 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Thu, 21 Oct 2021 12:13:43 +0200 Subject: [PATCH 11/31] ghc.json update --- public/version-aliases/ghc.json | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index ea4a3a9cc7..6b9da10600 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -115,9 +115,9 @@ "7.10.2": "7.10.2", "7.10.3": "7.10.3", "8.0.1": "8.0.1", - "8": "8.10.5", - "8.x": "8.10.5", - "8.x.x": "8.10.5", + "8": "8.10.7", + "8.x": "8.10.7", + "8.x.x": "8.10.7", "8.0.x": "8.0.2", "8.0": "8.0.2", "8.0.2": "8.0.2", @@ -153,14 +153,16 @@ "8.8.3": "8.8.3", "8.8.4": "8.8.4", "8.10.1-alpha1": "8.10.1-alpha1", - "8.10": "8.10.5", + "8.10": "8.10.7", "8.10.1-alpha2": "8.10.1-alpha2", "8.10.1": "8.10.1", - "8.10.x": "8.10.5", + "8.10.x": "8.10.7", "8.10.2": "8.10.2", "8.10.3": "8.10.3", "8.10.4": "8.10.4", "8.10.5": "8.10.5", + "8.10.6": "8.10.6", + "8.10.7": "8.10.7", "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.1", "9.0.1": "9.0.1", From 12d916e0c33f74c15752d1bac1134510985bdb9c Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Thu, 24 Mar 2022 10:18:45 +0100 Subject: [PATCH 12/31] Update travis_install_jdk.bash (#2013) --- lib/travis/build/bash/travis_install_jdk.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/build/bash/travis_install_jdk.bash b/lib/travis/build/bash/travis_install_jdk.bash index 1caa14e4f9..c1a3fffaf1 100644 --- a/lib/travis/build/bash/travis_install_jdk.bash +++ b/lib/travis/build/bash/travis_install_jdk.bash @@ -27,7 +27,7 @@ travis_install_jdk_ext_provider() { mkdir -p ~/bin url="https://$TRAVIS_APP_HOST/files/install-jdk.sh" if ! travis_download "$url" ~/bin/install-jdk.sh; then - url="https://raw.githubusercontent.com/sormuras/bach/master/install-jdk.sh" + url="https://raw.githubusercontent.com/sormuras/bach/releases/11/install-jdk.sh" travis_download "$url" ~/bin/install-jdk.sh || { echo "${ANSI_RED}Could not acquire install-jdk.sh. Stopping build.${ANSI_RESET}" >/dev/stderr travis_terminate 2 From 5f726233dd5967b4c767fa36d6fa5d937763a84a Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 30 Mar 2022 15:07:44 +0200 Subject: [PATCH 13/31] puma updated to 4.3.11 --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index baf7ea5aa8..2d86a0ce86 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -102,7 +102,7 @@ GEM net-http-persistent (3.0.0) connection_pool (~> 2.2) net-http-pipeline (1.0.1) - nio4r (2.5.7) + nio4r (2.5.8) octokit (4.18.0) faraday (>= 0.9) sawyer (~> 0.8.0, >= 0.5.3) @@ -117,7 +117,7 @@ GEM coderay (~> 1.1.0) method_source (~> 0.9.0) public_suffix (4.0.6) - puma (4.3.8) + puma (4.3.11) nio4r (~> 2.0) pusher-client (0.6.2) json From 5e5575ebbd9d135feed00d583c60648d0ab7be19 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Thu, 31 Mar 2022 12:40:13 +0200 Subject: [PATCH 14/31] puma update 4.3.12 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2d86a0ce86..b36fca0ae0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -117,7 +117,7 @@ GEM coderay (~> 1.1.0) method_source (~> 0.9.0) public_suffix (4.0.6) - puma (4.3.11) + puma (4.3.12) nio4r (~> 2.0) pusher-client (0.6.2) json From 9be13dacba60f7f0c2a7d1a85a875def9dc56ceb Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Thu, 31 Mar 2022 12:44:03 +0200 Subject: [PATCH 15/31] ghc up --- public/version-aliases/ghc.json | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 6b9da10600..879f3fd358 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -164,13 +164,17 @@ "8.10.6": "8.10.6", "8.10.7": "8.10.7", "9.0.1-alpha1": "9.0.1-alpha1", - "9.0": "9.0.1", + "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.0.1", - "9.x": "9.0.1", - "9.x.x": "9.0.1", - "9.0.x": "9.0.1", + "9": "9.2.2", + "9.x": "9.2.2", + "9.x.x": "9.2.2", + "9.0.x": "9.0.2", + "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", - "9.2": "9.2.1-alpha2", - "9.2.1-alpha2": "9.2.1-alpha2" + "9.2": "9.2.2", + "9.2.1-alpha2": "9.2.1-alpha2", + "9.2.1": "9.2.1", + "9.2.x": "9.2.2", + "9.2.2": "9.2.2" } \ No newline at end of file From 3c9d74d2dad2b01ae63656d84fb0f3783077664f Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Tue, 17 May 2022 13:50:16 +0200 Subject: [PATCH 16/31] updated image naming --- script/docker-build-and-push | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/docker-build-and-push b/script/docker-build-and-push index ba8f18a080..5986722bb7 100755 --- a/script/docker-build-and-push +++ b/script/docker-build-and-push @@ -15,8 +15,8 @@ docker-compose build "${app_name}" docker login -u="${QUAY_ROBOT_HANDLE}" -p="${QUAY_ROBOT_TOKEN}" quay.io docker images -docker tag "${local_image}" "${quay_image}:${TRAVIS_BRANCH}" -docker push "${quay_image}:${TRAVIS_BRANCH}" +docker tag "${local_image}" "${quay_image}:${TRAVIS_COMMIT:0:7}-${TRAVIS_BRANCH}" +docker push "${quay_image}:${TRAVIS_COMMIT:0:7}-${TRAVIS_BRANCH}" docker tag "${local_image}" "${quay_image}:${TRAVIS_COMMIT:0:7}" docker push "${quay_image}:${TRAVIS_COMMIT:0:7}" From 77912c4e9cbd07914758bc35fc414bda983a4812 Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Thu, 19 May 2022 11:16:04 +0200 Subject: [PATCH 17/31] gem updates 170522 (#2020) * gem updates * ghc update --- Gemfile | 2 +- Gemfile.lock | 23 +++++++++++------------ public/version-aliases/ghc.json | 4 +++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Gemfile b/Gemfile index b64427ece3..51f9c8effa 100644 --- a/Gemfile +++ b/Gemfile @@ -36,7 +36,7 @@ gem 'rspec', '~> 3.0', group: %i[development test] gem 'rubocop', require: false, group: %i[development test] gem 'sentry-raven' gem 'simplecov', require: false, group: %i[development test] -gem 'sinatra' +gem 'sinatra', '~> 2.2' gem 'sinatra-contrib' gem 'travis' gem 'travis-config' diff --git a/Gemfile.lock b/Gemfile.lock index b36fca0ae0..56da0196a0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,7 +47,7 @@ GEM ast (2.4.0) atomic (1.1.101) avl_tree (1.1.3) - backports (3.18.1) + backports (3.23.0) codeclimate-test-reporter (1.0.8) simplecov (<= 0.13) coder (0.4.0) @@ -95,7 +95,7 @@ GEM mocha (1.7.0) metaclass (~> 0.0.1) msgpack (1.2.4) - multi_json (1.14.1) + multi_json (1.15.0) multipart-post (2.1.1) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) @@ -123,7 +123,7 @@ GEM json websocket (~> 1.0) rack (2.2.3) - rack-protection (2.0.8.1) + rack-protection (2.2.0) rack rack-ssl (1.4.1) rack @@ -163,7 +163,7 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.10.0) - ruby2_keywords (0.0.2) + ruby2_keywords (0.0.5) ruby_dep (1.5.0) sawyer (0.8.2) addressable (>= 2.3.5) @@ -175,17 +175,16 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sinatra (2.0.8.1) + sinatra (2.2.0) mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.8.1) + rack (~> 2.2) + rack-protection (= 2.2.0) tilt (~> 2.0) - sinatra-contrib (2.0.8.1) - backports (>= 2.8.2) + sinatra-contrib (2.2.0) multi_json mustermann (~> 1.0) - rack-protection (= 2.0.8.1) - sinatra (= 2.0.8.1) + rack-protection (= 2.2.0) + sinatra (= 2.2.0) tilt (~> 2.0) thread_safe (0.3.6) tilt (2.0.10) @@ -238,7 +237,7 @@ DEPENDENCIES rubocop sentry-raven simplecov - sinatra + sinatra (~> 2.2) sinatra-contrib travis travis-config diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 879f3fd358..29c53dfb2b 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -176,5 +176,7 @@ "9.2.1-alpha2": "9.2.1-alpha2", "9.2.1": "9.2.1", "9.2.x": "9.2.2", - "9.2.2": "9.2.2" + "9.2.2": "9.2.2", + "9.4.1-alpha1": "9.4.1-alpha1", + "9.4": "9.4.1-alpha1" } \ No newline at end of file From f67d0395c629b5f1b527ddfdb7bd712c3c64a90e Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 1 Jun 2022 16:20:38 +0200 Subject: [PATCH 18/31] ghc update --- public/version-aliases/ghc.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 29c53dfb2b..20072ea8fd 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,17 +166,19 @@ "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.2.2", - "9.x": "9.2.2", - "9.x.x": "9.2.2", + "9": "9.2.3", + "9.x": "9.2.3", + "9.x.x": "9.2.3", "9.0.x": "9.0.2", "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", - "9.2": "9.2.2", + "9.2": "9.2.3", "9.2.1-alpha2": "9.2.1-alpha2", "9.2.1": "9.2.1", - "9.2.x": "9.2.2", + "9.2.x": "9.2.3", "9.2.2": "9.2.2", + "9.2.3": "9.2.3", "9.4.1-alpha1": "9.4.1-alpha1", - "9.4": "9.4.1-alpha1" + "9.4": "9.4.1-alpha2", + "9.4.1-alpha2": "9.4.1-alpha2" } \ No newline at end of file From b1317314854c2039a9d743e88b0b64dd29102b52 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 8 Jun 2022 11:13:53 +0200 Subject: [PATCH 19/31] rack update -> 2.2.3.1 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 56da0196a0..e803067e8e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -122,7 +122,7 @@ GEM pusher-client (0.6.2) json websocket (~> 1.0) - rack (2.2.3) + rack (2.2.3.1) rack-protection (2.2.0) rack rack-ssl (1.4.1) From 75e40542a107371309ac7c58b418451a0edea5b7 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 29 Jun 2022 13:10:17 +0200 Subject: [PATCH 20/31] ghc update --- public/version-aliases/ghc.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 20072ea8fd..3e019393d6 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -179,6 +179,7 @@ "9.2.2": "9.2.2", "9.2.3": "9.2.3", "9.4.1-alpha1": "9.4.1-alpha1", - "9.4": "9.4.1-alpha2", - "9.4.1-alpha2": "9.4.1-alpha2" + "9.4": "9.4.1-alpha3", + "9.4.1-alpha2": "9.4.1-alpha2", + "9.4.1-alpha3": "9.4.1-alpha3" } \ No newline at end of file From ffb7323644d4ac36c12243afacdecf2776b8d89f Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:26:17 +0200 Subject: [PATCH 21/31] extended access rights for gh tokens, using ssh in public repos if custom key is set (#2040) * github_apps update * force using ssh for public repos with custom keys (#2037) * ghc update --- Gemfile | 2 +- Gemfile.lock | 19 ++++++++++--------- lib/travis/build/data.rb | 8 ++++++-- public/version-aliases/ghc.json | 18 +++++++++++------- spec/build/data_spec.rb | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/Gemfile b/Gemfile index 51f9c8effa..67f4a04ec5 100644 --- a/Gemfile +++ b/Gemfile @@ -40,7 +40,7 @@ gem 'sinatra', '~> 2.2' gem 'sinatra-contrib' gem 'travis' gem 'travis-config' -gem 'travis-github_apps', git: gh('travis-ci/travis-github_apps') +gem 'travis-github_apps', git: 'https://github.com/travis-ci/travis-github_apps', branch: 'ga-ext_access' gem 'travis-rollout', git: gh('travis-ci/travis-rollout') gem 'travis-support', git: gh('travis-ci/travis-support') diff --git a/Gemfile.lock b/Gemfile.lock index e803067e8e..d5f90d2a55 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,7 +13,8 @@ GIT GIT remote: https://github.com/travis-ci/travis-github_apps - revision: c96dc9330849ff3e2ccd7c9d00005a1a96c1a4b6 + revision: 929dadf0b4f60ca4240d52a11fc032917ff3f83a + branch: ga-ext_access specs: travis-github_apps (0.2.1) activesupport (>= 3.2) @@ -37,7 +38,7 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (5.2.5) + activesupport (5.2.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -52,13 +53,13 @@ GEM simplecov (<= 0.13) coder (0.4.0) coderay (1.1.2) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.10) connection_pool (2.2.2) diff-lcs (1.3) docile (1.1.5) ethon (0.11.0) ffi (>= 1.3.0) - faraday (0.17.3) + faraday (0.17.5) multipart-post (>= 1.2, < 3) faraday_middleware (0.14.0) faraday (>= 0.7.4, < 1.0) @@ -73,7 +74,7 @@ GEM hashr (2.0.1) highline (1.7.10) hitimes (1.3.0) - i18n (1.8.10) + i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.1) json (2.3.1) @@ -91,12 +92,12 @@ GEM avl_tree (~> 1.1.2) hitimes (~> 1.1) minitar (0.6.1) - minitest (5.14.4) + minitest (5.15.0) mocha (1.7.0) metaclass (~> 0.0.1) msgpack (1.2.4) multi_json (1.15.0) - multipart-post (2.1.1) + multipart-post (2.2.3) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) net-http-persistent (3.0.0) @@ -138,7 +139,7 @@ GEM ffi (>= 1.0.6) msgpack (>= 0.4.3) optimist (>= 3.0.0) - redis (4.1.4) + redis (4.7.1) rerun (0.13.0) listen (~> 3.0) rspec (3.8.0) @@ -201,7 +202,7 @@ GEM hashr (~> 2.0) typhoeus (0.8.0) ethon (>= 0.8.0) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) unicode-display_width (1.4.0) websocket (1.2.8) diff --git a/lib/travis/build/data.rb b/lib/travis/build/data.rb index 30bfd6e306..4fc8a0686a 100644 --- a/lib/travis/build/data.rb +++ b/lib/travis/build/data.rb @@ -100,7 +100,7 @@ def ssh_key? def ssh_key @ssh_key ||= if ssh_key = data[:ssh_key] SshKey.new(ssh_key[:value], ssh_key[:source], ssh_key[:encoded]) - elsif source_key = data[:config][:source_key] + elsif data[:config] && source_key = data[:config][:source_key] SshKey.new(source_key, nil, true) end end @@ -144,13 +144,17 @@ def source_https? def source_ssh? return false if prefer_https? ((repo_private? || force_private?) && !installation?) || - (repo_private? && custom_ssh_key?) + ((repo_private? || enterprise?) && custom_ssh_key?) end def force_private? github? && !source_host&.include?('github.com') end + def enterprise? + ENV['TRAVIS_ENTERPRISE'] == 'true' || nil + end + def github? repository[:vcs_type] == 'GithubRepository' end diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index 3e019393d6..aae8379fb2 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,20 +166,24 @@ "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.2.3", - "9.x": "9.2.3", - "9.x.x": "9.2.3", + "9": "9.4.2", + "9.x": "9.4.2", + "9.x.x": "9.4.2", "9.0.x": "9.0.2", "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", - "9.2": "9.2.3", + "9.2": "9.2.4", "9.2.1-alpha2": "9.2.1-alpha2", "9.2.1": "9.2.1", - "9.2.x": "9.2.3", + "9.2.x": "9.2.4", "9.2.2": "9.2.2", "9.2.3": "9.2.3", + "9.2.4": "9.2.4", "9.4.1-alpha1": "9.4.1-alpha1", - "9.4": "9.4.1-alpha3", + "9.4": "9.4.2", "9.4.1-alpha2": "9.4.1-alpha2", - "9.4.1-alpha3": "9.4.1-alpha3" + "9.4.1-alpha3": "9.4.1-alpha3", + "9.4.1": "9.4.1", + "9.4.x": "9.4.2", + "9.4.2": "9.4.2" } \ No newline at end of file diff --git a/spec/build/data_spec.rb b/spec/build/data_spec.rb index dd30f259ee..e46a3e58b2 100644 --- a/spec/build/data_spec.rb +++ b/spec/build/data_spec.rb @@ -115,5 +115,37 @@ it { expect(data.source_ssh?).to be nil } it { expect(data.token).to eq 'access_token' } end + + describe 'source_ssh is true for public enterprise repository' do + let(:config) { {oauth_token: 'access_token', prefer_https: false, repository: { installation_id: 1, private: false, vcs_id: 123} } } + let(:data) { Travis::Build::Data.new(config) } + + before { + Travis::GithubApps.any_instance.stubs(:access_token).returns 'access_token' + ENV['TRAVIS_ENTERPRISE'] = 'true' + } + after { + ENV['TRAVIS_ENTERPRISE'] = nil + } + it { expect(data.installation?).to be true } + it { expect(data.source_ssh?).to be false } + it { expect(data.token).to eq 'access_token' } + end + + describe 'source_ssh is true for public enterprise repository if custom key is present' do + let(:config) { {oauth_token: 'access_token', prefer_https: false, repository: { installation_id: 1, private: false, vcs_id: 123}, ssh_key: { value: TEST_PRIVATE_KEY, source: 'repository_settings' } } } + let(:data) { Travis::Build::Data.new(config) } + + before { + Travis::GithubApps.any_instance.stubs(:access_token).returns 'access_token' + ENV['TRAVIS_ENTERPRISE'] = 'true' + } + after { + ENV['TRAVIS_ENTERPRISE'] = nil + } + it { expect(data.installation?).to be true } + it { expect(data.source_ssh?).to be true } + it { expect(data.token).to eq 'access_token' } + end end end From 2e3a4d420c519629da7d4af51c2764254b2398ab Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Wed, 5 Oct 2022 14:09:07 +0200 Subject: [PATCH 22/31] master merge, bab949e0107a (#2049) --- init.rb | 2 +- lib/travis/build/addons/apt.rb | 1 + lib/travis/build/addons/chrome.rb | 4 +- lib/travis/build/addons/mariadb.rb | 2 +- lib/travis/build/addons/rethinkdb.rb | 5 +- lib/travis/build/addons/snaps.rb | 1 + lib/travis/build/addons/ssh_known_hosts.rb | 1 + lib/travis/build/appliances.rb | 2 +- lib/travis/build/appliances/agent.rb | 1 - lib/travis/build/appliances/debug_tools.rb | 2 +- .../build/appliances/rm_etc_boto_cfg.rb | 1 - ...=> set_docker_mtu_and_registry_mirrors.rb} | 14 ++- lib/travis/build/appliances/setup_filter.rb | 1 - lib/travis/build/bash/travis_setup_env.bash | 2 +- .../build/bash/travis_setup_postgresql.bash | 3 + lib/travis/build/config.rb | 3 + lib/travis/build/errors.rb | 6 ++ lib/travis/build/rake_tasks.rb | 2 +- lib/travis/build/script.rb | 7 +- lib/travis/build/script/crystal.rb | 90 ++++++++----------- lib/travis/build/script/csharp.rb | 3 + lib/travis/build/script/matlab.rb | 62 +++++++++++++ lib/travis/build/script/perl.rb | 2 +- lib/travis/build/script/python.rb | 27 ++++++ lib/travis/build/script/r.rb | 30 ++++--- lib/travis/build/script/scala.rb | 2 +- lib/travis/build/script/shared/rvm.rb | 6 ++ spec/build/addons/apt_spec.rb | 2 +- spec/build/addons/mariadb_spec.rb | 2 +- spec/build/addons/rethinkdb_spec.rb | 4 +- spec/build/script/crystal_spec.rb | 53 +++++++---- spec/build/script/matlab_spec.rb | 43 +++++++++ spec/build/script/perl_spec.rb | 4 +- spec/build/script/r_spec.rb | 14 +-- spec/build/script/scala_spec.rb | 2 +- 35 files changed, 289 insertions(+), 117 deletions(-) rename lib/travis/build/appliances/{set_docker_mtu.rb => set_docker_mtu_and_registry_mirrors.rb} (54%) create mode 100644 lib/travis/build/script/matlab.rb create mode 100644 spec/build/script/matlab_spec.rb diff --git a/init.rb b/init.rb index 782b6c2ff8..b9babe0af7 100644 --- a/init.rb +++ b/init.rb @@ -74,7 +74,7 @@ def data end def set_up_config(match_data) - @build = build(match_data[:build]) + @build = build(match_data[:build].to_i) @job_number = match_data[:job].to_i - 1 @compile_config = @build.jobs[@job_number].config end diff --git a/lib/travis/build/addons/apt.rb b/lib/travis/build/addons/apt.rb index afe6435d46..2ebc4029da 100644 --- a/lib/travis/build/addons/apt.rb +++ b/lib/travis/build/addons/apt.rb @@ -15,6 +15,7 @@ class Apt < Base xenial bionic focal + jammy ).freeze attr_reader :safelisted, :disallowed_while_sudo diff --git a/lib/travis/build/addons/chrome.rb b/lib/travis/build/addons/chrome.rb index 8829210d3a..7382126449 100644 --- a/lib/travis/build/addons/chrome.rb +++ b/lib/travis/build/addons/chrome.rb @@ -60,9 +60,9 @@ def export_source_url sh.elif "$(uname) = 'Darwin'" do case version when 'stable' - pkg_url = "https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg" + pkg_url = "https://dl.google.com/chrome/mac/universal/stable/GGRO/googlechrome.dmg" when 'beta' - pkg_url = "https://dl.google.com/chrome/mac/beta/googlechrome.dmg" + pkg_url = "https://dl.google.com/chrome/mac/universal/beta/googlechromebeta.dmg " end sh.export 'CHROME_SOURCE_URL', pkg_url end diff --git a/lib/travis/build/addons/mariadb.rb b/lib/travis/build/addons/mariadb.rb index 84845875ac..ee76f0e3d1 100644 --- a/lib/travis/build/addons/mariadb.rb +++ b/lib/travis/build/addons/mariadb.rb @@ -21,7 +21,7 @@ def after_prepare sh.else do sh.cmd "apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 #{MARIADB_GPG_KEY_NEW}", sudo: true end - sh.cmd 'add-apt-repository "deb http://%p/mariadb/repo/%p/ubuntu $TRAVIS_DIST main"' % [MARIADB_MIRROR, mariadb_version], sudo: true + sh.cmd 'add-apt-repository --yes "deb http://%p/mariadb/repo/%p/ubuntu $TRAVIS_DIST main"' % [MARIADB_MIRROR, mariadb_version], sudo: true sh.cmd 'travis_apt_get_update', retry: true, echo: true sh.cmd "PACKAGES='mariadb-server-#{mariadb_version}'", echo: true sh.cmd "if [[ $(lsb_release -cs) = 'precise' ]]; then PACKAGES=\"${PACKAGES} libmariadbclient-dev\"; fi", echo: true diff --git a/lib/travis/build/addons/rethinkdb.rb b/lib/travis/build/addons/rethinkdb.rb index 41b6193a6e..3537b7dee2 100644 --- a/lib/travis/build/addons/rethinkdb.rb +++ b/lib/travis/build/addons/rethinkdb.rb @@ -15,8 +15,8 @@ def after_prepare sh.else do sh.echo "Installing RethinkDB version #{rethinkdb_version}", ansi: :yellow sh.cmd "service rethinkdb stop", sudo: true - sh.cmd "wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -v -''", echo: true - sh.cmd 'echo -e "\ndeb http://download.rethinkdb.com/apt $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list > /dev/null' + sh.cmd "sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys \"539A 3A8C 6692 E6E3 F69B 3FE8 1D85 E93F 801B B43F\"", echo: true + sh.cmd 'echo -e "\ndeb https://download.rethinkdb.com/repository/ubuntu-$(lsb_release -cs)/ $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list.d/rethinkdb.list > /dev/null' sh.cmd 'travis_apt_get_update', assert: false sh.cmd "package_version=`apt-cache show rethinkdb | grep -F \"Version: #{rethinkdb_version}\" | sort -r | head -n 1 | awk '{printf $2}'`" sh.cmd "apt-get install -y -o Dpkg::Options::='--force-confnew' rethinkdb=$package_version", sudo: true, echo: true, timing: true @@ -39,4 +39,3 @@ def rethinkdb_version end end end - diff --git a/lib/travis/build/addons/snaps.rb b/lib/travis/build/addons/snaps.rb index 682393661b..ec9f667c05 100644 --- a/lib/travis/build/addons/snaps.rb +++ b/lib/travis/build/addons/snaps.rb @@ -13,6 +13,7 @@ class Snaps < Base xenial bionic focal + jammy ).freeze def before_prepare? diff --git a/lib/travis/build/addons/ssh_known_hosts.rb b/lib/travis/build/addons/ssh_known_hosts.rb index 643deeadc4..c1288d823c 100644 --- a/lib/travis/build/addons/ssh_known_hosts.rb +++ b/lib/travis/build/addons/ssh_known_hosts.rb @@ -25,6 +25,7 @@ def config def add_ssh_known_hosts sh.fold 'ssh_known_hosts.0' do sh.echo "Adding ssh known hosts", ansi: :yellow + sh.mkdir "${TRAVIS_HOME}/.ssh", recursive: true config.each do |host| begin host_uri = URI("ssh://#{host}") diff --git a/lib/travis/build/appliances.rb b/lib/travis/build/appliances.rb index b1eee38b7a..e389d5eb59 100644 --- a/lib/travis/build/appliances.rb +++ b/lib/travis/build/appliances.rb @@ -36,7 +36,7 @@ require 'travis/build/appliances/rvm_use' require 'travis/build/appliances/services' require 'travis/build/appliances/show_system_info' -require 'travis/build/appliances/set_docker_mtu' +require 'travis/build/appliances/set_docker_mtu_and_registry_mirrors' require 'travis/build/appliances/set_x' require 'travis/build/appliances/setup_filter' require 'travis/build/appliances/shell_session_update' diff --git a/lib/travis/build/appliances/agent.rb b/lib/travis/build/appliances/agent.rb index 9a3a26dc4c..5279ab8dd0 100644 --- a/lib/travis/build/appliances/agent.rb +++ b/lib/travis/build/appliances/agent.rb @@ -1,7 +1,6 @@ require 'base64' require 'travis/build/appliances/base' require 'travis/build/appliances/agent/jwt' -require 'travis/build/git' module Travis module Build diff --git a/lib/travis/build/appliances/debug_tools.rb b/lib/travis/build/appliances/debug_tools.rb index 92116553c5..2c1a0b1f2b 100644 --- a/lib/travis/build/appliances/debug_tools.rb +++ b/lib/travis/build/appliances/debug_tools.rb @@ -69,7 +69,7 @@ def install_dir # XXX the following does not apply to OSX def static_build_linux_url - if config[:arch] == 'arm64' + if config[:arch] =~ /^arm64\b/ "https://#{app_host}/files/tmate-static-linux-arm64v8.tar.xz" else "https://#{app_host}/files/tmate-static-linux-amd64.tar.xz" diff --git a/lib/travis/build/appliances/rm_etc_boto_cfg.rb b/lib/travis/build/appliances/rm_etc_boto_cfg.rb index 1779f519b1..c27c174f45 100644 --- a/lib/travis/build/appliances/rm_etc_boto_cfg.rb +++ b/lib/travis/build/appliances/rm_etc_boto_cfg.rb @@ -1,5 +1,4 @@ require 'travis/build/appliances/base' -require 'travis/build/git' module Travis module Build diff --git a/lib/travis/build/appliances/set_docker_mtu.rb b/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb similarity index 54% rename from lib/travis/build/appliances/set_docker_mtu.rb rename to lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb index ca693f0add..58b7aa77c6 100644 --- a/lib/travis/build/appliances/set_docker_mtu.rb +++ b/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb @@ -3,13 +3,16 @@ module Travis module Build module Appliances - class SetDockerMtu < Base + class SetDockerMtuAndRegistryMirrors < Base + + REGISTRY_URL = Travis::Build.config.registry_url.output_safe.freeze + def apply? linux? end def apply - sh.fold "docker_mtu" do + sh.fold "docker_mtu_and_registry_mirrors" do sh.raw <<-EOF sudo test -f /etc/docker/daemon.json if [[ $? = 0 ]]; then @@ -20,6 +23,13 @@ def apply echo '{"mtu":1460}' | sudo tee /etc/docker/daemon.json > /dev/null fi +if curl --connect-timeout 1 -fsSL -o /dev/null \ + "#{REGISTRY_URL}" &>/dev/null; then + echo '[{"op":"add","path":"/registry-mirrors","value":["#{REGISTRY_URL}"]}]' > registry.jsonpatch + sudo jsonpatch /etc/docker/daemon.json registry.jsonpatch > daemon.json + sudo mv daemon.json /etc/docker/daemon.json +fi + sudo service docker restart EOF end diff --git a/lib/travis/build/appliances/setup_filter.rb b/lib/travis/build/appliances/setup_filter.rb index 222d35fb12..d173e7cc00 100644 --- a/lib/travis/build/appliances/setup_filter.rb +++ b/lib/travis/build/appliances/setup_filter.rb @@ -1,5 +1,4 @@ require 'travis/build/appliances/base' -require 'travis/build/git' require 'travis/rollout' module Travis diff --git a/lib/travis/build/bash/travis_setup_env.bash b/lib/travis/build/bash/travis_setup_env.bash index aaa5982611..69384dcbc0 100644 --- a/lib/travis/build/bash/travis_setup_env.bash +++ b/lib/travis/build/bash/travis_setup_env.bash @@ -76,7 +76,7 @@ travis_setup_env() { export TRAVIS_TEST_RESULT= export TRAVIS_CMD= - TRAVIS_TMPDIR="$(mktemp -d 2>/dev/null || mktemp -d -t 'travis_tmp')" + TRAVIS_TMPDIR="$(mktemp -d 2>/dev/null || mktemp -d -t travis_tmp.XXXX)" mkdir -p "${TRAVIS_TMPDIR}" export TRAVIS_TMPDIR diff --git a/lib/travis/build/bash/travis_setup_postgresql.bash b/lib/travis/build/bash/travis_setup_postgresql.bash index ebe59f6bed..399fa4878a 100644 --- a/lib/travis/build/bash/travis_setup_postgresql.bash +++ b/lib/travis/build/bash/travis_setup_postgresql.bash @@ -19,6 +19,9 @@ travis_setup_postgresql() { focal) version='12' ;; + jammy) + version='14' + ;; *) echo -e "${ANSI_RED}Unrecognized operating system.${ANSI_CLEAR}" ;; diff --git a/lib/travis/build/config.rb b/lib/travis/build/config.rb index 6dcdb0ea02..8670cb2466 100644 --- a/lib/travis/build/config.rb +++ b/lib/travis/build/config.rb @@ -53,6 +53,7 @@ def sc_data xenial: ENV.fetch('TRAVIS_BUILD_APT_PACKAGE_SAFELIST_XENIAL', ''), bionic: ENV.fetch('TRAVIS_BUILD_APT_PACKAGE_SAFELIST_BIONIC', ''), focal: ENV.fetch('TRAVIS_BUILD_APT_PACKAGE_SAFELIST_FOCAL', ''), + jammy: ENV.fetch('TRAVIS_BUILD_APT_PACKAGE_SAFELIST_JAMMY', ''), }, apt_proxy: ENV.fetch('TRAVIS_BUILD_APT_PROXY', ''), apt_source_alias_list: { @@ -61,6 +62,7 @@ def sc_data xenial: ENV.fetch('TRAVIS_BUILD_APT_SOURCE_ALIAS_LIST_XENIAL', ''), bionic: ENV.fetch('TRAVIS_BUILD_APT_SOURCE_ALIAS_LIST_BIONIC', ''), focal: ENV.fetch('TRAVIS_BUILD_APT_SOURCE_ALIAS_LIST_FOCAL', ''), + jammy: ENV.fetch('TRAVIS_BUILD_APT_SOURCE_ALIAS_LIST_JAMMY', ''), }, apt_source_alias_list_key_url_template: ENV.fetch( 'TRAVIS_BUILD_APT_SOURCE_ALIAS_LIST_KEY_URL_TEMPLATE', @@ -105,6 +107,7 @@ def sc_data ), }, maven_central_mirror: ENV.fetch('TRAVIS_MAVEN_CENTRAL_MIRROR', ''), + registry_url: ENV.fetch('TRAVIS_BUILD_REGISTRY_URL', 'https://registry.travis-ci.com'), network: { wait_retries: Integer(ENV.fetch( 'TRAVIS_BUILD_NETWORK_WAIT_RETRIES', diff --git a/lib/travis/build/errors.rb b/lib/travis/build/errors.rb index 970230581f..f24ad4f711 100644 --- a/lib/travis/build/errors.rb +++ b/lib/travis/build/errors.rb @@ -89,5 +89,11 @@ def initialize(msg = "Unable to fetch GitHub Apps Token. GitHub may be unavailab super end end + + class UnknownServiceTypeError < StandardError + def initialize(_type) + super + end + end end end diff --git a/lib/travis/build/rake_tasks.rb b/lib/travis/build/rake_tasks.rb index b917485777..c1fbc1c07f 100644 --- a/lib/travis/build/rake_tasks.rb +++ b/lib/travis/build/rake_tasks.rb @@ -330,7 +330,7 @@ def file_update_ghc_versions dest.chmod(0o644) end - def file_update_sonar_scanner(version: ENV['TRAVIS_BUILD_SONAR_CLOUD_CLI_VERSION'] || '3.0.3.778') + def file_update_sonar_scanner(version: ENV['TRAVIS_BUILD_SONAR_CLOUD_CLI_VERSION'] || '4.7.0.2747') conn = build_faraday_conn(host: 'repo1.maven.org') response = conn.get("/maven2/org/sonarsource/scanner/cli/sonar-scanner-cli/#{version}/sonar-scanner-cli-#{version}.zip") raise 'Could not fetch SonarCloud scanner CLI archive' unless response.success? diff --git a/lib/travis/build/script.rb b/lib/travis/build/script.rb index 0b68489498..7943d74a4d 100644 --- a/lib/travis/build/script.rb +++ b/lib/travis/build/script.rb @@ -28,6 +28,7 @@ require 'travis/build/script/haskell' require 'travis/build/script/haxe' require 'travis/build/script/julia' +require 'travis/build/script/matlab' require 'travis/build/script/nix' require 'travis/build/script/node_js' require 'travis/build/script/elm' @@ -339,7 +340,7 @@ def configure apply :update_heroku apply :shell_session_update apply :git_v2 - apply :set_docker_mtu + apply :set_docker_mtu_and_registry_mirrors apply :resolvconf apply :maven_central_mirror apply :maven_https @@ -481,7 +482,7 @@ def use_workspaces sh.fold "workspaces_use" do ws_names.each do |name| - sh.echo "Fetching workspace #{name}", ansi: :green + sh.echo "Fetching workspace #{shesc(name)}", ansi: :green ws = Travis::Build::Script::Workspace.new(sh, data, name, [], :use) ws.install_casher ws.fetch @@ -507,7 +508,7 @@ def create_workspaces "or an array of such hashes", ansi: :yellow next end - sh.echo "Workspace: #{cfg[:name]}", ansi: :green + sh.echo "Workspace: #{shesc(cfg[:name])}", ansi: :green ws = Travis::Build::Script::Workspace.new(sh, data, cfg[:name], cfg[:paths], :create) ws.install_casher ws.compress diff --git a/lib/travis/build/script/crystal.rb b/lib/travis/build/script/crystal.rb index 6a46a146ec..8b009d2b43 100644 --- a/lib/travis/build/script/crystal.rb +++ b/lib/travis/build/script/crystal.rb @@ -3,7 +3,7 @@ module Build class Script class Crystal < Script DEFAULTS = { - crystal: 'latest', + crystal: 'stable', } def configure @@ -14,14 +14,10 @@ def configure case config[:os] when 'linux' - validate_version - if crystal_config_version == 'nightly' - linux_nightly - else - linux_latest - end + validate_crystal_config + apt_install_crystal when 'osx' - if crystal_config_version != "latest" + if crystal_config != "latest" && crystal_config != "stable" sh.failure %Q(Specifying Crystal version is not yet supported by the macOS environment) end sh.cmd %q(brew update) @@ -68,64 +64,53 @@ def setup_cache end def cache_slug - super << '-crystal-' << crystal_config_version + super << '-crystal-' << crystal_config end private - def crystal_config_version + def crystal_config Array(config[:crystal]).first.to_s end - def validate_version - if crystal_config_version != 'latest' && crystal_config_version != 'nightly' - sh.failure %Q("#{crystal_config_version}" is an invalid version of Crystal.\nView valid versions of Crystal at https://docs.travis-ci.com/user/languages/crystal/) - end + def validate_crystal_config + # - stable + # - latest (same as stable, backward compatibility) + # - unstable + # - nightly + # - x.y (from stable channel) + # - x.y.z (from stable channel) + # - /x.y (where = stable, unstable, nightly) + # - /x.y.z (where = stable, unstable, nightly) + return if crystal_config == "latest" + return if crystal_config =~ /\A(stable|unstable|nightly)(\/(\d+)(\.\d+)(\.\d+)?)?/ + return if crystal_config =~ /\A(\d+)(\.\d+)(\.\d+)?/ + + sh.failure %Q("#{crystal_config}" is an invalid version of Crystal.\nView valid versions of Crystal at https://docs.travis-ci.com/user/languages/crystal/) end - def linux_latest - sh.if "-n $(command -v snap)" do - snap_install_crystal '--channel=latest/stable' - end - sh.else do - apt_install_crystal - end - end + def apt_install_crystal + config = crystal_config + config = "stable" if config == "latest" - def linux_nightly - sh.if "-n $(command -v snap)" do - snap_install_crystal '--channel=latest/edge' - end - sh.else do - sh.failure "Crystal nightlies will only be supported via snap. Use Xenial or later releases." + if config =~ /\A(\d+)(\.\d+)(\.\d+)?/ + crystal_channel = "stable" + crystal_version = config + else + crystal_channel, crystal_version = config.split('/') + crystal_version ||= "latest" end - end - - def snap_install_crystal(options) - sh.cmd 'travis_apt_get_update' - sh.cmd %Q(sudo apt-get install -y gcc pkg-config git tzdata libpcre3-dev libevent-dev libyaml-dev libgmp-dev libssl-dev libxml2-dev 2>&1 > /dev/null), echo: true - sh.cmd %Q(sudo snap install crystal --classic #{options}), echo: true - end - def apt_install_crystal - version = { - url: "https://dist.crystal-lang.org/apt", - key: { - url: "https://dist.crystal-lang.org/rpm/RPM-GPG-KEY", - fingerprint: "5995C83CD754BE448164192909617FD37CC06B54" - }, - package: "crystal" - } - - sh.cmd %Q(curl -sSL '#{version[:key][:url]}' > "${TRAVIS_HOME}/crystal_repository_key.asc") - sh.if %Q("$(gpg --with-fingerprint "${TRAVIS_HOME}/crystal_repository_key.asc" | grep "Key fingerprint" | cut -d "=" -f2 | tr -d " ")" != "#{version[:key][:fingerprint]}") do - sh.failure "The repository key needed to install Crystal did not have the expected fingerprint. Your build was aborted." - end - sh.cmd %q(sudo sh -c "apt-key add '${TRAVIS_HOME}/crystal_repository_key.asc'") + # Add repo metadata signign key (shared bintray signing key) + sh.cmd %q(sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61) + sh.cmd %Q(echo "deb https://dl.bintray.com/crystal/deb all #{crystal_channel}" | sudo tee /etc/apt/sources.list.d/crystal.list) - sh.cmd %Q(sudo sh -c 'echo "deb #{version[:url]} crystal main" > /etc/apt/sources.list.d/crystal-nightly.list') sh.cmd 'travis_apt_get_update' - sh.cmd %Q(sudo apt-get install -y #{version[:package]} libgmp-dev) + if crystal_version == "latest" + sh.cmd %Q(sudo apt-get install -y crystal) + else + sh.cmd %Q(sudo apt-get install -y crystal="#{crystal_version}*") + end end def cache_dirs @@ -133,7 +118,6 @@ def cache_dirs when 'linux' %W( ${TRAVIS_HOME}/.cache/shards - ${TRAVIS_HOME}/snap/crystal/common/.cache/shards ) when 'osx' %W( diff --git a/lib/travis/build/script/csharp.rb b/lib/travis/build/script/csharp.rb index f2eab1c22b..c746e17d8b 100644 --- a/lib/travis/build/script/csharp.rb +++ b/lib/travis/build/script/csharp.rb @@ -145,6 +145,9 @@ def install_dotnet sh.elif '$(lsb_release -cs) = bionic' do sh.cmd "sudo sh -c \"echo 'deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main' > /etc/apt/sources.list.d/dotnet-official.list\"", assert: true end + sh.elif '$(lsb_release -cs) = focal' do + sh.cmd "sudo sh -c \"echo 'deb [arch=amd64] https://packages.microsoft.com/ubuntu/20.04/prod focal main' > /etc/apt/sources.list.d/dotnet-official.list\"", assert: true + end sh.else do sh.failure "The version of this operating system is not supported by .NET Core. View valid versions at https://docs.travis-ci.com/user/languages/csharp/" end diff --git a/lib/travis/build/script/matlab.rb b/lib/travis/build/script/matlab.rb new file mode 100644 index 0000000000..bfee9970ed --- /dev/null +++ b/lib/travis/build/script/matlab.rb @@ -0,0 +1,62 @@ +module Travis + module Build + class Script + class Matlab < Script + MATLAB_INSTALLER_LOCATION = 'https://ssd.mathworks.com/supportfiles/ci/ephemeral-matlab/v0/ci-install.sh'.freeze + MATLAB_DEPS_LOCATION = 'https://ssd.mathworks.com/supportfiles/ci/matlab-deps/v0/install.sh'.freeze + MATLAB_START = 'matlab -batch'.freeze + MATLAB_COMMAND = "assertSuccess(runtests('IncludeSubfolders',true));".freeze + + MATLAB_NOTICE = [ + 'The MATLAB language is maintained by MathWorks.', + 'If you have any questions or suggestions, please contact MathWorks at continuous-integration@mathworks.com.', + ] + + DEFAULTS = { + matlab: 'latest' + }.freeze + + def export + super + sh.export 'TRAVIS_MATLAB_VERSION', release.shellescape, echo: false + end + + def setup + super + + # Echo support notice + MATLAB_NOTICE.each do |message| + sh.echo message, ansi: :green + end + + sh.fold 'matlab_install' do + sh.echo 'Installing MATLAB', ansi: :yellow + + # Execute helper script to install runtime dependencies + sh.echo 'Installing system dependencies', ansi: :yellow + sh.raw "wget -qO- --retry-connrefused #{MATLAB_DEPS_LOCATION}" \ + ' | sudo -E bash -s -- $TRAVIS_MATLAB_VERSION' + + # Invoke the ephemeral MATLAB installer that will make a MATLAB available + # on the system PATH + sh.echo 'Setting up MATLAB', ansi: :yellow + sh.raw "wget -qO- --retry-connrefused #{MATLAB_INSTALLER_LOCATION}" \ + ' | sudo -E bash -s -- --release $TRAVIS_MATLAB_VERSION' + end + end + + def script + super + # By default, invoke the default MATLAB 'runtests' command + sh.cmd "#{MATLAB_START} \"#{MATLAB_COMMAND}\"" + end + + private + + def release + Array(config[:matlab]).first.to_s + end + end + end + end +end diff --git a/lib/travis/build/script/perl.rb b/lib/travis/build/script/perl.rb index d2dd27b452..2af3df4060 100644 --- a/lib/travis/build/script/perl.rb +++ b/lib/travis/build/script/perl.rb @@ -27,7 +27,7 @@ def setup def announce super sh.cmd 'perl --version' - sh.cmd 'cpanm --version' + sh.cmd 'CP_VER=$(cpanm --version &); echo $CP_VER' end def install diff --git a/lib/travis/build/script/python.rb b/lib/travis/build/script/python.rb index 1bb9043cb1..eb2eebd7cf 100644 --- a/lib/travis/build/script/python.rb +++ b/lib/travis/build/script/python.rb @@ -16,6 +16,13 @@ class Python < Script } ] + PIP_20_3_MSG = [ + "Pip version 20.3 introduces changes to the dependency resolver that may affect your software.", + "We advise you to consider testing the upcoming changes, which may be introduced in a future Travis CI build image update.", + "See https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-2-2020 for more information." + ] + PIP_20_2_MSG = "With pip 20.2, you can test the new dependency resolver with the \\\`--use-feature=2020-resolver\\\` flag." + REQUIREMENTS_MISSING = 'Could not locate requirements.txt. Override the install: key in your .travis.yml to install dependencies.' SCRIPT_MISSING = 'Please override the script: key in your .travis.yml to run tests.' @@ -57,6 +64,7 @@ def setup_cache end def install + warn_pip_20_3 sh.if '-f Requirements.txt' do sh.cmd 'pip install -r Requirements.txt', fold: 'install', retry: true end @@ -139,6 +147,25 @@ def tar_extract def setup_path(version = 'nightly') sh.cmd "echo 'export PATH=/opt/python/#{version}/bin:$PATH' | sudo tee -a #{PYENV_PATH_FILE} &>/dev/null" end + + def pip_version_at_least_20_2? + "$(travis_vers2int $(pip --version | cut -f2 -d \" \")) -ge $(travis_vers2int \"20.2\")" + end + def pip_version_before_20_3? + "$(travis_vers2int $(pip --version | cut -f2 -d \" \")) -lt $(travis_vers2int \"20.3\")" + end + + def warn_pip_20_3 + sh.if pip_version_before_20_3? do + PIP_20_3_MSG.each { |l| sh.echo l, ansi: :yellow } + + sh.if pip_version_at_least_20_2? do + sh.echo PIP_20_2_MSG, ansi: :yellow + end + + sh.echo + end + end end end end diff --git a/lib/travis/build/script/r.rb b/lib/travis/build/script/r.rb index ac0d090490..ee049afc91 100644 --- a/lib/travis/build/script/r.rb +++ b/lib/travis/build/script/r.rb @@ -67,6 +67,13 @@ def configure sh.echo 'Installing R', ansi: :yellow case config[:os] when 'linux' + if config[:arch] == 'arm64' + sh.failure 'ARM architecture not supported' + end + if config[:dist] == 'trusty' + sh.failure '"dist: trusty" is no longer supported for "language: r"' + end + # This key is added implicitly by the marutter PPA below #sh.cmd 'apt-key adv --keyserver ha.pool.sks-keyservers.net '\ #'--recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9', sudo: true @@ -87,11 +94,6 @@ def configure sh.cmd 'sudo add-apt-repository -y "ppa:ubuntugis/ppa"' sh.cmd 'sudo add-apt-repository -y "ppa:cran/travis"' - # Both c2d4u and c2d4u3.5 depend on this ppa for ffmpeg - sh.if "$(lsb_release -cs) = 'trusty'" do - sh.cmd 'sudo add-apt-repository -y "ppa:kirillshkrogalev/ffmpeg-next"' - end - # Update after adding all repositories. Retry several # times to work around flaky connection to Launchpad PPAs. sh.cmd 'travis_apt_get_update', retry: true @@ -112,12 +114,13 @@ def configure 'cdbs qpdf texinfo libssh2-1-dev devscripts '\ "#{optional_apt_pkgs}", retry: true - r_filename = "R-#{r_version}-$(lsb_release -cs).xz" - r_url = "https://travis-ci.rstudio.org/#{r_filename}" + r_filename = "r-#{r_version}_1_amd64.deb" + os_version = "$(lsb_release -rs | tr -d '.')" + r_url = "https://cdn.rstudio.com/r/ubuntu-#{os_version}/pkgs/#{r_filename}" sh.cmd "curl -fLo /tmp/#{r_filename} #{r_url}", retry: true - sh.cmd "tar xJf /tmp/#{r_filename} -C ~" - sh.export 'PATH', "${TRAVIS_HOME}/R-bin/bin:$PATH", echo: false - sh.export 'LD_LIBRARY_PATH', "${TRAVIS_HOME}/R-bin/lib:$LD_LIBRARY_PATH", echo: false + sh.cmd "sudo apt-get install -y gdebi-core" + sh.cmd "sudo gdebi --non-interactive /tmp/#{r_filename}" + sh.export 'PATH', "/opt/R/#{r_version}/bin:$PATH", echo: false sh.rm "/tmp/#{r_filename}" sh.cmd "sudo mkdir -p /usr/local/lib/R/site-library $R_LIBS_USER" @@ -178,7 +181,6 @@ def configure config[:r_build_args] = config[:r_build_args] + " --no-manual" end - setup_bioc if needs_bioc? setup_pandoc if config[:pandoc] # Removes preinstalled homebrew @@ -202,6 +204,8 @@ def install sh.failure "No DESCRIPTION file found, user must supply their own install and script steps" end + setup_bioc if needs_bioc? + sh.fold "R-dependencies" do sh.echo 'Installing package dependencies', ansi: :yellow @@ -604,7 +608,7 @@ def r_version_less_than(str) def normalized_r_version(v=Array(config[:r]).first.to_s) case v - when 'release' then '4.0.0' + when 'release' then '4.0.2' when 'oldrel' then '3.6.3' when '3.0' then '3.0.3' when '3.1' then '3.1.3' @@ -613,7 +617,7 @@ def normalized_r_version(v=Array(config[:r]).first.to_s) when '3.4' then '3.4.4' when '3.5' then '3.5.3' when '3.6' then '3.6.3' - when '4.0' then '4.0.0' + when '4.0' then '4.0.2' when 'bioc-devel' config[:bioc_required] = true config[:bioc_use_devel] = true diff --git a/lib/travis/build/script/scala.rb b/lib/travis/build/script/scala.rb index 7df8b15ec6..4a3a3f21cb 100644 --- a/lib/travis/build/script/scala.rb +++ b/lib/travis/build/script/scala.rb @@ -11,7 +11,7 @@ class Scala < Jvm } SBT_PATH = '/usr/local/bin/sbt' - SBT_SHA = '4ad1b8a325f75c1a66f3fd100635da5eb28d9c91' + SBT_SHA = '4d558c88ae6ae240e7a5eedc0cf33cc9e7bd0e58' SBT_URL = "https://raw.githubusercontent.com/paulp/sbt-extras/#{SBT_SHA}/sbt" def configure diff --git a/lib/travis/build/script/shared/rvm.rb b/lib/travis/build/script/shared/rvm.rb index a739882143..3e4b76964b 100644 --- a/lib/travis/build/script/shared/rvm.rb +++ b/lib/travis/build/script/shared/rvm.rb @@ -121,6 +121,9 @@ def use_default_ruby def use_ruby_version_file sh.fold('rvm') do + sh.if '-n $(grep "^3" .ruby-version)' do + sh.cmd 'rvm get head' + end sh.cmd 'rvm use $(< .ruby-version) --install --binary --fuzzy' end end @@ -156,6 +159,9 @@ def use_ruby_version sh.cmd "rvm use #{ruby_version} --install --binary --fuzzy" end else + if ruby_version.start_with? '3' + sh.cmd "rvm get head" + end sh.cmd "rvm use #{ruby_version} --install --binary --fuzzy" end end diff --git a/spec/build/addons/apt_spec.rb b/spec/build/addons/apt_spec.rb index 4434ce8010..86bfb48f06 100644 --- a/spec/build/addons/apt_spec.rb +++ b/spec/build/addons/apt_spec.rb @@ -109,7 +109,7 @@ it 'defaults source safelist to empty hash' do expect(described_class.source_alias_lists) - .to eql({ unset: {}, precise: {}, trusty: {}, xenial: {}, bionic: {}, focal: {} }) + .to eql({ unset: {}, precise: {}, trusty: {}, xenial: {}, bionic: {}, focal: {}, jammy: {} }) end end diff --git a/spec/build/addons/mariadb_spec.rb b/spec/build/addons/mariadb_spec.rb index a4938018c3..f3a16df0e9 100644 --- a/spec/build/addons/mariadb_spec.rb +++ b/spec/build/addons/mariadb_spec.rb @@ -21,7 +21,7 @@ it { should include_sexp [:cmd, "service mysql stop", sudo: true] } it { should include_sexp [:cmd, "apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 #{Travis::Build::Addons::Mariadb::MARIADB_GPG_KEY_OLD}", sudo: true] } - it { should include_sexp [:cmd, 'add-apt-repository "deb http://%p/mariadb/repo/%p/ubuntu $TRAVIS_DIST main"' % [Travis::Build::Addons::Mariadb::MARIADB_MIRROR, config], sudo: true] } + it { should include_sexp [:cmd, 'add-apt-repository --yes "deb http://%p/mariadb/repo/%p/ubuntu $TRAVIS_DIST main"' % [Travis::Build::Addons::Mariadb::MARIADB_MIRROR, config], sudo: true] } it { should include_sexp [:cmd, 'travis_apt_get_update', retry: true, echo: true] } it { should include_sexp [:cmd, "PACKAGES='mariadb-server-10.0'", echo: true] } it { should include_sexp [:cmd, "rm -rf /var/lib/mysql", sudo: true] } diff --git a/spec/build/addons/rethinkdb_spec.rb b/spec/build/addons/rethinkdb_spec.rb index 1880630683..381ec555ab 100644 --- a/spec/build/addons/rethinkdb_spec.rb +++ b/spec/build/addons/rethinkdb_spec.rb @@ -24,8 +24,8 @@ end it { should include_sexp [:cmd, "service rethinkdb stop", sudo: true] } - it { should include_sexp [:cmd, "wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -v -''", echo: true] } - it { should include_sexp [:cmd, 'echo -e "\ndeb http://download.rethinkdb.com/apt $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list > /dev/null'] } + it { should include_sexp [:cmd, "sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys \"539A 3A8C 6692 E6E3 F69B 3FE8 1D85 E93F 801B B43F\"", echo: true] } + it { should include_sexp [:cmd, 'echo -e "\ndeb https://download.rethinkdb.com/repository/ubuntu-$(lsb_release -cs)/ $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list.d/rethinkdb.list > /dev/null'] } it { should include_sexp [:cmd, 'travis_apt_get_update'] } it { should include_sexp [:cmd, "apt-get install -y -o Dpkg::Options::='--force-confnew' rethinkdb=$package_version", sudo: true, echo: true, timing: true] } it { should include_sexp [:cmd, "cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/default.conf", sudo: true] } diff --git a/spec/build/script/crystal_spec.rb b/spec/build/script/crystal_spec.rb index 20e21c5653..52560a1cbc 100644 --- a/spec/build/script/crystal_spec.rb +++ b/spec/build/script/crystal_spec.rb @@ -25,17 +25,14 @@ describe '#cache_slug' do subject { described_class.new(data).cache_slug } - it { is_expected.to eq("cache-#{CACHE_SLUG_EXTRAS}-crystal-latest") } + it { is_expected.to eq("cache-#{CACHE_SLUG_EXTRAS}-crystal-stable") } end context "versions" do - let(:with_snap) { sexp_find(subject, [:if, '-n $(command -v snap)'], [:then]) } - let(:without_snap) { sexp_find(subject, [:if, '-n $(command -v snap)'], [:else]) } - it "installs latest linux release by default" do data[:config][:os] = "linux" - expect(with_snap).to include_sexp [:cmd, "sudo snap install crystal --classic --channel=latest/stable", {echo: true}] - expect(without_snap).to include_sexp [:cmd, "sudo apt-get install -y crystal libgmp-dev"] + should include_sexp [:cmd, %q(echo "deb https://dl.bintray.com/crystal/deb all stable" | sudo tee /etc/apt/sources.list.d/crystal.list)] + should include_sexp [:cmd, "sudo apt-get install -y crystal"] end it "installs latest macOS release by default" do @@ -43,17 +40,38 @@ should include_sexp [:cmd, "brew install crystal-lang"] end - it "installs latest linux release when explicitly asked for" do + it "installs latest stable linux release (with crystal: latest)" do data[:config][:os] = "linux" data[:config][:crystal] = "latest" - expect(with_snap).to include_sexp [:cmd, "sudo snap install crystal --classic --channel=latest/stable", {echo: true}] + should include_sexp [:cmd, %q(echo "deb https://dl.bintray.com/crystal/deb all stable" | sudo tee /etc/apt/sources.list.d/crystal.list)] + should include_sexp [:cmd, "sudo apt-get install -y crystal"] end - it "installs linux nightly when specified" do - data[:config][:os] = "linux" - data[:config][:crystal] = "nightly" - expect(with_snap).to include_sexp [:cmd, "sudo snap install crystal --classic --channel=latest/edge", {echo: true}] - expect(without_snap).to include_sexp [:echo, "Crystal nightlies will only be supported via snap. Use Xenial or later releases."] + %w(stable unstable nightly).each do |channel| + it "installs latest stable linux release (with crystal: #{channel})" do + data[:config][:os] = "linux" + data[:config][:crystal] = channel + should include_sexp [:cmd, %Q(echo "deb https://dl.bintray.com/crystal/deb all #{channel}" | sudo tee /etc/apt/sources.list.d/crystal.list)] + should include_sexp [:cmd, "sudo apt-get install -y crystal"] + end + + %w(0.35 1.0.1 1.1.0-pre3).each do |version| + it "installs specific channel/version linux release (with crystal: #{channel}/#{version})" do + data[:config][:os] = "linux" + data[:config][:crystal] = "#{channel}/#{version}" + should include_sexp [:cmd, %Q(echo "deb https://dl.bintray.com/crystal/deb all #{channel}" | sudo tee /etc/apt/sources.list.d/crystal.list)] + should include_sexp [:cmd, %Q(sudo apt-get install -y crystal="#{version}*")] + end + end + end + + %w(0.35 1.0.1 1.1.0-pre3).each do |version| + it "installs specific stable version release (with crystal: #{version})" do + data[:config][:os] = "linux" + data[:config][:crystal] = version + should include_sexp [:cmd, %Q(echo "deb https://dl.bintray.com/crystal/deb all stable" | sudo tee /etc/apt/sources.list.d/crystal.list)] + should include_sexp [:cmd, %Q(sudo apt-get install -y crystal="#{version}*")] + end end it 'throws a error with a non-release version on macOS' do @@ -67,9 +85,12 @@ should include_sexp [:echo, "Operating system not supported: \"invalid\""] end - it 'throws a error with a invalid version' do - data[:config][:crystal] = "foo" - should include_sexp [:echo, "\"foo\" is an invalid version of Crystal.\nView valid versions of Crystal at https://docs.travis-ci.com/user/languages/crystal/"] + %w(foo wrong1.0.0 notstable).each do |invalid_version| + # wrong1.0.0 and notstable were choosed to check valid values as suffix + it "throws a error with a invalid version (with crystal: #{invalid_version}" do + data[:config][:crystal] = invalid_version + should include_sexp [:echo, "\"#{invalid_version}\" is an invalid version of Crystal.\nView valid versions of Crystal at https://docs.travis-ci.com/user/languages/crystal/"] + end end end end diff --git a/spec/build/script/matlab_spec.rb b/spec/build/script/matlab_spec.rb new file mode 100644 index 0000000000..7593b2cbff --- /dev/null +++ b/spec/build/script/matlab_spec.rb @@ -0,0 +1,43 @@ +require 'spec_helper' + +describe Travis::Build::Script::Matlab, :sexp do + let(:data) { payload_for(:push, :matlab) } + let(:script) { described_class.new(data) } + let(:installer) { Travis::Build::Script::Matlab::MATLAB_INSTALLER_LOCATION } + let(:helper) { Travis::Build::Script::Matlab::MATLAB_DEPS_LOCATION } + let(:start) { Travis::Build::Script::Matlab::MATLAB_START } + let(:command) { Travis::Build::Script::Matlab::MATLAB_COMMAND } + let(:notice) { Travis::Build::Script::Matlab::MATLAB_NOTICE } + + subject { script.sexp } + it { store_example } + + it_behaves_like 'a bash script' + + it 'sets TRAVIS_MATLAB_VERSION to the latest version of MATLAB' do + should include_sexp [:export, %w[TRAVIS_MATLAB_VERSION latest]] + end + + it 'prints the support notice in green' do + notice.each do |message| + should include_sexp [:echo, message, ansi: :green] + end + end + + it 'configures runtime dependencies' do + should include_sexp [:raw, "wget -qO- --retry-connrefused #{helper}"\ + ' | sudo -E bash -s -- $TRAVIS_MATLAB_VERSION', assert: true] + end + + context 'it sets up MATLAB' do + it 'by calling the ephemeral installer script' do + should include_sexp [:raw, "wget -qO- --retry-connrefused #{installer}"\ + ' | sudo -E bash -s -- --release $TRAVIS_MATLAB_VERSION', assert: true] + end + end + + it 'runs the default MATLAB command' do + should include_sexp [:cmd, "#{start} \"#{command}\"", + echo: true, timing: true] + end +end diff --git a/spec/build/script/perl_spec.rb b/spec/build/script/perl_spec.rb index 9c7fc7919a..b08acfe8f9 100644 --- a/spec/build/script/perl_spec.rb +++ b/spec/build/script/perl_spec.rb @@ -42,8 +42,8 @@ should include_sexp [:cmd, 'perl --version', echo: true] end - it 'announces cpanm --version' do - should include_sexp [:cmd, 'cpanm --version', echo: true] + it 'announces CP_VER=$(cpanm --version &); echo $CP_VER' do + should include_sexp [:cmd, 'CP_VER=$(cpanm --version &); echo $CP_VER', echo: true] end it 'installs' do diff --git a/spec/build/script/r_spec.rb b/spec/build/script/r_spec.rb index 546eda934b..22bd55ac53 100644 --- a/spec/build/script/r_spec.rb +++ b/spec/build/script/r_spec.rb @@ -11,7 +11,7 @@ it 'normalizes bioc-devel correctly' do data[:config][:r] = 'bioc-devel' - should include_sexp [:export, ['TRAVIS_R_VERSION', '4.0.0']] + should include_sexp [:export, ['TRAVIS_R_VERSION', '4.0.2']] should include_sexp [:cmd, %r{install.packages\(\"BiocManager"\)}, assert: true, echo: true, timing: true, retry: true] should include_sexp [:cmd, %r{BiocManager::install\(version = \"devel\"}, @@ -22,7 +22,7 @@ data[:config][:r] = 'bioc-release' should include_sexp [:cmd, %r{install.packages\(\"BiocManager"\)}, assert: true, echo: true, timing: true, retry: true] - should include_sexp [:export, ['TRAVIS_R_VERSION', '4.0.0']] + should include_sexp [:export, ['TRAVIS_R_VERSION', '4.0.2']] end it 'r_packages works with a single package set' do @@ -50,7 +50,7 @@ end it 'downloads and installs latest R' do - should include_sexp [:cmd, %r{^curl.*https://travis-ci\.rstudio\.org/R-4\.0\.0-\$\(lsb_release -cs\)\.xz}, + should include_sexp [:cmd, %r{^curl.*https://cdn.rstudio.com/r/ubuntu-.*/pkgs/r-4\.0\.2_1_amd64\.deb}, assert: true, echo: true, retry: true, timing: true] end @@ -104,19 +104,19 @@ it 'downloads and installs R 3.1' do data[:config][:r] = '3.1' - should include_sexp [:cmd, %r{^curl.*https://travis-ci\.rstudio\.org/R-3\.1\.3-\$\(lsb_release -cs\)\.xz}, + should include_sexp [:cmd, %r{^curl.*https://cdn.rstudio.com/r/ubuntu-.*/pkgs/r-3\.1\.3_1_amd64\.deb}, assert: true, echo: true, retry: true, timing: true] end it 'downloads and installs R 3.2' do data[:config][:r] = '3.2' - should include_sexp [:cmd, %r{^curl.*https://travis-ci\.rstudio\.org/R-3\.2\.5-\$\(lsb_release -cs\)\.xz}, + should include_sexp [:cmd, %r{^curl.*https://cdn.rstudio.com/r/ubuntu-.*/pkgs/r-3\.2\.5_1_amd64\.deb}, assert: true, echo: true, retry: true, timing: true] end it 'downloads and installs R devel' do data[:config][:r] = 'devel' - should include_sexp [:cmd, %r{^curl.*https://travis-ci\.rstudio\.org/R-devel-\$\(lsb_release -cs\)\.xz}, + should include_sexp [:cmd, %r{^curl.*https://cdn.rstudio.com/r/ubuntu-.*/pkgs/r-devel_1_amd64\.deb}, assert: true, echo: true, retry: true, timing: true] end @@ -257,7 +257,7 @@ } it { data[:config][:r] = 'release' - should eq("cache-#{CACHE_SLUG_EXTRAS}--R-4.0.0") + should eq("cache-#{CACHE_SLUG_EXTRAS}--R-4.0.2") } it { data[:config][:r] = 'oldrel' diff --git a/spec/build/script/scala_spec.rb b/spec/build/script/scala_spec.rb index 15a6cd228f..5a5e7dbb59 100644 --- a/spec/build/script/scala_spec.rb +++ b/spec/build/script/scala_spec.rb @@ -4,7 +4,7 @@ let(:data) { payload_for(:push, :scala) } let(:script) { described_class.new(data) } let(:sbt_path) { '/usr/local/bin/sbt'} - let(:sbt_sha) { '4ad1b8a325f75c1a66f3fd100635da5eb28d9c91'} + let(:sbt_sha) { '4d558c88ae6ae240e7a5eedc0cf33cc9e7bd0e58'} let(:sbt_url) { "https://build.travis-ci.org/files/sbt"} before do From a31d7cc480b5d7c90ebdbecd7fd29d35b8eae493 Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:25:20 +0100 Subject: [PATCH 23/31] merge from master (#2054) * merge from master 20.10.2022 * spec fixes, merge fixes ship:docker * ghc update --- Gemfile | 2 + Gemfile.lock | 27 ++++ lib/travis/build.rb | 7 +- lib/travis/build/addons/artifacts/env.rb | 3 +- lib/travis/build/addons/deploy/script.rb | 6 +- lib/travis/build/appliances.rb | 2 + lib/travis/build/appliances/checkout.rb | 4 +- lib/travis/build/appliances/enable_i386.rb | 2 +- lib/travis/build/appliances/setup_filter.rb | 4 +- lib/travis/build/appliances/vault_connect.rb | 27 ++++ lib/travis/build/appliances/vault_keys.rb | 25 +++ lib/travis/build/bash/travis_install_jdk.bash | 83 ++++++---- lib/travis/build/data.rb | 8 + lib/travis/build/script.rb | 15 +- lib/travis/build/script/clojure.rb | 1 + lib/travis/build/script/shared/rvm.rb | 14 +- lib/travis/services/vault.rb | 9 ++ lib/travis/services/vault/connect.rb | 14 ++ lib/travis/services/vault/keys.rb | 25 +++ lib/travis/services/vault/keys/build_paths.rb | 35 ++++ lib/travis/services/vault/keys/kv1.rb | 16 ++ lib/travis/services/vault/keys/kv2.rb | 16 ++ lib/travis/services/vault/keys/paths.rb | 15 ++ lib/travis/services/vault/keys/resolver.rb | 49 ++++++ lib/travis/services/vault/keys/version.rb | 13 ++ lib/travis/vcs.rb | 72 +++++++++ lib/travis/vcs/base.rb | 34 ++++ lib/travis/{build => vcs}/git.rb | 65 +++++--- lib/travis/{build => vcs}/git/clone.rb | 6 +- lib/travis/{build => vcs}/git/netrc.rb | 4 +- lib/travis/{build => vcs}/git/ssh_key.rb | 4 +- lib/travis/{build => vcs}/git/submodules.rb | 4 +- lib/travis/{build => vcs}/git/tarball.rb | 4 +- lib/travis/vcs/perforce.rb | 149 ++++++++++++++++++ lib/travis/vcs/perforce/clone.rb | 125 +++++++++++++++ lib/travis/vcs/perforce/netrc.rb | 31 ++++ lib/travis/vcs/perforce/ssh_key.rb | 39 +++++ lib/travis/vcs/perforce/submodules.rb | 18 +++ lib/travis/vcs/perforce/tarball.rb | 57 +++++++ lib/travis/vcs/svn.rb | 149 ++++++++++++++++++ lib/travis/vcs/svn/clone.rb | 105 ++++++++++++ lib/travis/vcs/svn/netrc.rb | 31 ++++ lib/travis/vcs/svn/ssh_key.rb | 45 ++++++ lib/travis/vcs/svn/submodules.rb | 18 +++ lib/travis/vcs/svn/tarball.rb | 57 +++++++ public/version-aliases/ghc.json | 18 ++- spec/build/appliances/vault_connect_spec.rb | 130 +++++++++++++++ spec/build/appliances/vault_keys_spec.rb | 79 ++++++++++ spec/build/git/clone_spec.rb | 4 +- spec/build/git/ssh_key.rb | 2 +- spec/build/git/submodules_spec.rb | 2 +- spec/build/git/tarball_spec.rb | 2 +- spec/build/git_spec.rb | 2 +- spec/build/services/vault/connect_spec.rb | 45 ++++++ .../services/vault/keys/build_paths_spec.rb | 112 +++++++++++++ spec/build/services/vault/keys/kv1_spec.rb | 40 +++++ spec/build/services/vault/keys/kv2_spec.rb | 40 +++++ spec/build/services/vault/keys/paths_spec.rb | 27 ++++ .../services/vault/keys/resolver_spec.rb | 63 ++++++++ .../build/services/vault/keys/version_spec.rb | 36 +++++ spec/build/services/vault/keys_spec.rb | 24 +++ spec/build/vault_spec.rb | 59 +++++++ .../fixtures/build_config_with_vault_kv1.json | 113 +++++++++++++ .../fixtures/build_config_with_vault_kv2.json | 114 ++++++++++++++ spec/spec_helper.rb | 3 + spec/support/payloads.rb | 6 +- 66 files changed, 2259 insertions(+), 101 deletions(-) create mode 100644 lib/travis/build/appliances/vault_connect.rb create mode 100644 lib/travis/build/appliances/vault_keys.rb create mode 100644 lib/travis/services/vault.rb create mode 100644 lib/travis/services/vault/connect.rb create mode 100644 lib/travis/services/vault/keys.rb create mode 100644 lib/travis/services/vault/keys/build_paths.rb create mode 100644 lib/travis/services/vault/keys/kv1.rb create mode 100644 lib/travis/services/vault/keys/kv2.rb create mode 100644 lib/travis/services/vault/keys/paths.rb create mode 100644 lib/travis/services/vault/keys/resolver.rb create mode 100644 lib/travis/services/vault/keys/version.rb create mode 100644 lib/travis/vcs.rb create mode 100644 lib/travis/vcs/base.rb rename lib/travis/{build => vcs}/git.rb (66%) rename lib/travis/{build => vcs}/git/clone.rb (99%) rename lib/travis/{build => vcs}/git/netrc.rb (96%) rename lib/travis/{build => vcs}/git/ssh_key.rb (97%) rename lib/travis/{build => vcs}/git/submodules.rb (95%) rename lib/travis/{build => vcs}/git/tarball.rb (97%) create mode 100644 lib/travis/vcs/perforce.rb create mode 100644 lib/travis/vcs/perforce/clone.rb create mode 100644 lib/travis/vcs/perforce/netrc.rb create mode 100644 lib/travis/vcs/perforce/ssh_key.rb create mode 100644 lib/travis/vcs/perforce/submodules.rb create mode 100644 lib/travis/vcs/perforce/tarball.rb create mode 100644 lib/travis/vcs/svn.rb create mode 100644 lib/travis/vcs/svn/clone.rb create mode 100644 lib/travis/vcs/svn/netrc.rb create mode 100644 lib/travis/vcs/svn/ssh_key.rb create mode 100644 lib/travis/vcs/svn/submodules.rb create mode 100644 lib/travis/vcs/svn/tarball.rb create mode 100644 spec/build/appliances/vault_connect_spec.rb create mode 100644 spec/build/appliances/vault_keys_spec.rb create mode 100644 spec/build/services/vault/connect_spec.rb create mode 100644 spec/build/services/vault/keys/build_paths_spec.rb create mode 100644 spec/build/services/vault/keys/kv1_spec.rb create mode 100644 spec/build/services/vault/keys/kv2_spec.rb create mode 100644 spec/build/services/vault/keys/paths_spec.rb create mode 100644 spec/build/services/vault/keys/resolver_spec.rb create mode 100644 spec/build/services/vault/keys/version_spec.rb create mode 100644 spec/build/services/vault/keys_spec.rb create mode 100644 spec/build/vault_spec.rb create mode 100644 spec/fixtures/build_config_with_vault_kv1.json create mode 100644 spec/fixtures/build_config_with_vault_kv2.json diff --git a/Gemfile b/Gemfile index 67f4a04ec5..555c674a31 100644 --- a/Gemfile +++ b/Gemfile @@ -25,6 +25,7 @@ gem 'minitar' gem 'mocha', require: false, group: %i[development test] gem 'parallel_tests', require: false, group: %i[development test] gem 'pry', require: false, group: %i[development test] +gem 'webmock', group: :test gem 'puma', '~> 4' gem 'rack', '>= 2.1.4' gem 'rack-ssl', '~> 1.4' @@ -45,3 +46,4 @@ gem 'travis-rollout', git: gh('travis-ci/travis-rollout') gem 'travis-support', git: gh('travis-ci/travis-support') gem "octokit", "~> 4.18" +gem 'rest-client' diff --git a/Gemfile.lock b/Gemfile.lock index d5f90d2a55..683e29b4ec 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -55,8 +55,12 @@ GEM coderay (1.1.2) concurrent-ruby (1.1.10) connection_pool (2.2.2) + crack (0.4.5) + rexml diff-lcs (1.3) docile (1.1.5) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) ethon (0.11.0) ffi (>= 1.3.0) faraday (0.17.5) @@ -71,9 +75,13 @@ GEM multi_json (~> 1.0) net-http-persistent (>= 2.7) net-http-pipeline + hashdiff (1.0.1) hashr (2.0.1) highline (1.7.10) hitimes (1.3.0) + http-accept (1.7.0) + http-cookie (1.0.5) + domain_name (~> 0.5) i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.1) @@ -91,6 +99,9 @@ GEM atomic (~> 1.0) avl_tree (~> 1.1.2) hitimes (~> 1.1) + mime-types (3.4.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2022.0105) minitar (0.6.1) minitest (5.15.0) mocha (1.7.0) @@ -103,6 +114,7 @@ GEM net-http-persistent (3.0.0) connection_pool (~> 2.2) net-http-pipeline (1.0.1) + netrc (0.11.0) nio4r (2.5.8) octokit (4.18.0) faraday (>= 0.9) @@ -142,6 +154,12 @@ GEM redis (4.7.1) rerun (0.13.0) listen (~> 3.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.5) rspec (3.8.0) rspec-core (~> 3.8.0) rspec-expectations (~> 3.8.0) @@ -204,7 +222,14 @@ GEM ethon (>= 0.8.0) tzinfo (1.2.10) thread_safe (~> 0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) unicode-display_width (1.4.0) + webmock (3.18.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) websocket (1.2.8) PLATFORMS @@ -234,6 +259,7 @@ DEPENDENCIES rake rbtrace rerun + rest-client rspec (~> 3.0) rubocop sentry-raven @@ -245,6 +271,7 @@ DEPENDENCIES travis-github_apps! travis-rollout! travis-support! + webmock BUNDLED WITH 1.17.3 diff --git a/lib/travis/build.rb b/lib/travis/build.rb index cc8e20f1c7..9f15a66e17 100644 --- a/lib/travis/build.rb +++ b/lib/travis/build.rb @@ -16,9 +16,7 @@ def config module_function :config def top - @top ||= Pathname.new( - `git rev-parse --show-toplevel 2>/dev/null`.strip - ) + ::Travis::Vcs::Git::top end module_function :top @@ -26,7 +24,7 @@ def top class << self def version return @version if @version - @version ||= `git describe --always --dirty --tags 2>/dev/null`.strip + @version ||= ::Travis::Vcs::Git::version @version = nil unless $?.success? @version ||= ENV.fetch('BUILD_SLUG_COMMIT', nil) @version ||= top.join('VERSION').read if top.join('VERSION').exist? @@ -80,3 +78,4 @@ def logger require 'travis/build/data' require 'travis/build/env' require 'travis/build/script' +require 'travis/services/vault' diff --git a/lib/travis/build/addons/artifacts/env.rb b/lib/travis/build/addons/artifacts/env.rb index 44a0c79604..b573640df2 100644 --- a/lib/travis/build/addons/artifacts/env.rb +++ b/lib/travis/build/addons/artifacts/env.rb @@ -1,5 +1,6 @@ require 'core_ext/hash/deep_symbolize_keys' require 'active_support/core_ext/hash/slice' +require 'travis/vcs.rb' module Travis module Build @@ -7,7 +8,7 @@ class Addons class Artifacts < Base class Env DEFAULT = { - paths: '$(git ls-files -o | tr "\n" ":")', + paths: ::Travis::Vcs::paths, log_format: 'multiline' } diff --git a/lib/travis/build/addons/deploy/script.rb b/lib/travis/build/addons/deploy/script.rb index 7f86a5d75d..aa04455506 100644 --- a/lib/travis/build/addons/deploy/script.rb +++ b/lib/travis/build/addons/deploy/script.rb @@ -267,10 +267,10 @@ def build_gem_locally_from(source, branch) sh.echo "Building dpl gem locally with source #{source} and branch #{branch}", ansi: :yellow cmd("gem uninstall -aIx dpl >& /dev/null", echo: false, assert: !allow_failure, timing: false) sh.cmd("pushd /tmp >& /dev/null", echo: false, assert: !allow_failure, timing: true) - sh.cmd("git clone https://github.com/#{source} #{source}", echo: true, assert: !allow_failure, timing: true) + sh.cmd(::Travis::Vcs::Git::clone_cmd('https://github.com',source), echo: true, assert: !allow_failure, timing: true) sh.cmd("pushd #{source} >& /dev/null", echo: false, assert: !allow_failure, timing: true) - sh.cmd("git checkout #{branch}", echo: true, assert: !allow_failure, timing: true) - sh.cmd("git rev-parse HEAD", echo: true, assert: !allow_failure, timing: true) + sh.cmd(::Travis::Vcs::Git::checkout_cmd(branch), echo: true, assert: !allow_failure, timing: true) + sh.cmd(::Travis::Vcs::Git::revision_cmd, echo: true, assert: !allow_failure, timing: true) cmd("gem build dpl.gemspec", echo: true, assert: !allow_failure, timing: true) sh.raw "for f in dpl-*.gemspec; do" sh.raw " base=${f%*.gemspec}" diff --git a/lib/travis/build/appliances.rb b/lib/travis/build/appliances.rb index e389d5eb59..0b395f14bb 100644 --- a/lib/travis/build/appliances.rb +++ b/lib/travis/build/appliances.rb @@ -54,6 +54,8 @@ require 'travis/build/appliances/resolvconf' require 'travis/build/appliances/maven_central_mirror' require 'travis/build/appliances/maven_https' +require 'travis/build/appliances/vault_connect' +require 'travis/build/appliances/vault_keys' module Travis module Build diff --git a/lib/travis/build/appliances/checkout.rb b/lib/travis/build/appliances/checkout.rb index ba390d34ae..94d9e2cedf 100644 --- a/lib/travis/build/appliances/checkout.rb +++ b/lib/travis/build/appliances/checkout.rb @@ -1,12 +1,12 @@ require 'travis/build/appliances/base' -require 'travis/build/git' +require 'travis/vcs' module Travis module Build module Appliances class Checkout < Base def apply - Git.new(sh, data).checkout + Travis::Vcs.checkout(sh, data) end def apply? diff --git a/lib/travis/build/appliances/enable_i386.rb b/lib/travis/build/appliances/enable_i386.rb index 2ecfecb2aa..aff12329f7 100644 --- a/lib/travis/build/appliances/enable_i386.rb +++ b/lib/travis/build/appliances/enable_i386.rb @@ -5,7 +5,7 @@ module Build module Appliances class EnableI386 < Base def apply - sh.if "$(uname -m) == x86_64 && $(command -v lsb_release) && $(lsb_release -cs) != precise" do + sh.if "$(uname -m) == x86_64 && $(command -v lsb_release) && $(lsb_release -cs) != precise && $(lsb_release -cs) != Ootpa" do sh.cmd 'dpkg --add-architecture i386', echo: false, assert: false, sudo: true end end diff --git a/lib/travis/build/appliances/setup_filter.rb b/lib/travis/build/appliances/setup_filter.rb index d173e7cc00..9850ffc601 100644 --- a/lib/travis/build/appliances/setup_filter.rb +++ b/lib/travis/build/appliances/setup_filter.rb @@ -63,7 +63,7 @@ def blocklist def apply? sh.echo 'Secret environment variables are not obfuscated on Windows, please refer to our documentation: https://docs.travis-ci.com/user/best-practices-security', ansi: 'yellow' if windows? - enabled? and secrets.any? and !windows? + enabled? && secrets.any? && !windows? end def apply @@ -105,7 +105,7 @@ def exports end def secrets - @secrets ||= env_secrets.concat(data.secrets).uniq + @secrets ||= env_secrets.concat(data.secrets).concat(data.vault_secrets).uniq end def env_secrets diff --git a/lib/travis/build/appliances/vault_connect.rb b/lib/travis/build/appliances/vault_connect.rb new file mode 100644 index 0000000000..785dfbcaaa --- /dev/null +++ b/lib/travis/build/appliances/vault_connect.rb @@ -0,0 +1,27 @@ +require 'travis/build/appliances/base' +require 'travis/services/vault' + +module Travis + module Build + module Appliances + class VaultConnect < Base + ERROR_MESSAGE = ["Failed to connect to the Vault instance. Please verify if:\n* The Vault Token is correct (encrypted, not plain text). \n* The Vault Token is not expired. \n* The Vault can accept connections from the Travis CI build job environments (https://docs.travis-ci.com/user/ip-addresses/).", ansi: :red].freeze + SUCCESS_MESSAGE = ['Connected to Vault instance.', ansi: :green].freeze + + def apply? + @vault = config[:vault] if (config.dig(:vault, :secrets) || []).flatten.any? { |el| el.is_a?(String) } + end + + def apply + Travis::Vault::Connect.call(@vault) + sh.echo *SUCCESS_MESSAGE + sh.export('VAULT_ADDR', @vault[:api_url], echo: true, secure: true) + sh.export('VAULT_TOKEN', @vault[:token], echo: true, secure: true) + rescue Travis::Vault::ConnectionError, ArgumentError, URI::InvalidURIError => _e + sh.echo *ERROR_MESSAGE + sh.terminate + end + end + end + end +end diff --git a/lib/travis/build/appliances/vault_keys.rb b/lib/travis/build/appliances/vault_keys.rb new file mode 100644 index 0000000000..4f7f6b7eef --- /dev/null +++ b/lib/travis/build/appliances/vault_keys.rb @@ -0,0 +1,25 @@ +require 'travis/build/appliances/base' +require 'travis/services/vault' + +module Travis + module Build + module Appliances + class VaultKeys < Base + ERROR_MESSAGE = ['Too many keys in fetched data. Probably you provided the root key. Terminating for security reasons.', ansi: :red].freeze + + attr_reader :vault + + def apply? + @vault = config[:vault] if config.dig(:vault, :secrets).present? + end + + def apply + Travis::Vault::Keys.new(self).resolve + rescue Travis::Vault::RootKeyError + sh.echo *ERROR_MESSAGE + sh.terminate + end + end + end + end +end diff --git a/lib/travis/build/bash/travis_install_jdk.bash b/lib/travis/build/bash/travis_install_jdk.bash index c1a3fffaf1..f0117c339c 100644 --- a/lib/travis/build/bash/travis_install_jdk.bash +++ b/lib/travis/build/bash/travis_install_jdk.bash @@ -1,49 +1,45 @@ travis_install_jdk() { + # shellcheck disable=SC2034 local url vendor version license jdk certlink + # shellcheck disable=SC2034 jdk="$1" + # shellcheck disable=SC2034 vendor="$2" + # shellcheck disable=SC2034 version="$3" case "${TRAVIS_CPU_ARCH}" in - "arm64" | "s390x" | "ppc64le") - travis_install_jdk_package "$version" + "s390x" | "ppc64le") + travis_install_jdk_package_adoptopenjdk "$version" ;; - *) - travis_install_jdk_ext_provider "$jdk" "$vendor" "$version" + "amd64") + case "${TRAVIS_DIST}" in + "trusty") + travis_jdk_trusty "$version" + ;; + *) + travis_install_jdk_package_bellsoft "$version" + ;; + esac + ;; + "arm64") + travis_install_jdk_package_bellsoft "$version" ;; esac } -travis_install_jdk_ext_provider() { - local url vendor version license jdk certlink - jdk="$1" - vendor="$2" - version="$3" - if [[ "$vendor" == openjdk ]]; then - license=GPL - elif [[ "$vendor" == oracle ]]; then - license=BCL - fi - mkdir -p ~/bin - url="https://$TRAVIS_APP_HOST/files/install-jdk.sh" - if ! travis_download "$url" ~/bin/install-jdk.sh; then - url="https://raw.githubusercontent.com/sormuras/bach/releases/11/install-jdk.sh" - travis_download "$url" ~/bin/install-jdk.sh || { - echo "${ANSI_RED}Could not acquire install-jdk.sh. Stopping build.${ANSI_RESET}" >/dev/stderr - travis_terminate 2 - } - fi - chmod +x ~/bin/install-jdk.sh - travis_cmd "export JAVA_HOME=~/$jdk" --echo - # shellcheck disable=SC2016 - travis_cmd 'export PATH="$JAVA_HOME/bin:$PATH"' --echo - [[ "$TRAVIS_OS_NAME" == linux && "$vendor" == openjdk ]] && certlink=" --cacerts" - # shellcheck disable=2088 - travis_cmd "~/bin/install-jdk.sh --target \"$JAVA_HOME\" --workspace \"$TRAVIS_HOME/.cache/install-jdk\" --feature \"$version\" --license \"$license\"$certlink" --echo --assert +# Trusty image issues with new jdk provider +travis_jdk_trusty() { + local JAVA_VERSION + JAVA_VERSION="$1" + sudo apt-get update -yqq + PACKAGE="java-${JAVA_VERSION}-openjdk-amd64" + sudo apt install openjdk-"$JAVA_VERSION"-jdk + travis_cmd "export JAVA_HOME=/usr/lib/jvm/$PACKAGE" --echo + travis_cmd "export PATH=$JAVA_HOME/bin:$PATH" --echo } -travis_install_jdk_package() { - +travis_install_jdk_package_adoptopenjdk() { local JAVA_VERSION JAVA_VERSION="$1" sudo apt-get update -yqq @@ -52,6 +48,29 @@ travis_install_jdk_package() { if dpkg-query -l adoptopenjdk* >/dev/null 2>&1; then dpkg-query -l adoptopenjdk* | grep adoptopenjdk | awk '{print $2}' | xargs sudo dpkg -P fi + wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - + sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ + sudo apt-get update -yqq + sudo apt-get -yqq --no-install-suggests --no-install-recommends install "$PACKAGE" || true + sudo update-java-alternatives -s "$PACKAGE"* + fi +} + +travis_install_jdk_package_bellsoft() { + local JAVA_VERSION + JAVA_VERSION="$1" + sudo apt-get update -yqq + if [[ "$JAVA_VERSION" == "8" ]]; then + JAVA_VERSION="1.8.0" + fi + PACKAGE="bellsoft-java${JAVA_VERSION}" + if ! dpkg -s "$PACKAGE" >/dev/null 2>&1; then + wget -qO - https://download.bell-sw.com/pki/GPG-KEY-bellsoft | sudo apt-key add - + sudo add-apt-repository "deb [arch=$TRAVIS_CPU_ARCH] https://apt.bell-sw.com/ stable main" + sudo apt-get update -yqq sudo apt-get -yqq --no-install-suggests --no-install-recommends install "$PACKAGE" || true + travis_cmd "export JAVA_HOME=/usr/lib/jvm/bellsoft-java${JAVA_VERSION}-${TRAVIS_CPU_ARCH}" --echo + travis_cmd "export PATH=$JAVA_HOME/bin:$PATH" --echo + sudo update-java-alternatives -s "$PACKAGE"* fi } diff --git a/lib/travis/build/data.rb b/lib/travis/build/data.rb index 4fc8a0686a..e342a3106e 100644 --- a/lib/travis/build/data.rb +++ b/lib/travis/build/data.rb @@ -125,6 +125,14 @@ def secrets Array(data[:secrets]) end + def vault_secrets=(v_secrets) + data[:vault_secrets] = Array(v_secrets) + end + + def vault_secrets + Array(data[:vault_secrets]) + end + def disable_sudo? !!data[:paranoid] end diff --git a/lib/travis/build/script.rb b/lib/travis/build/script.rb index 7943d74a4d..3b48537ffa 100644 --- a/lib/travis/build/script.rb +++ b/lib/travis/build/script.rb @@ -8,7 +8,7 @@ require 'travis/build/addons' require 'travis/build/appliances' require 'travis/build/errors' -require 'travis/build/git' +require 'travis/vcs' require 'travis/build/helpers' require 'travis/build/stages' @@ -84,13 +84,13 @@ class Script private_constant :TRAVIS_FUNCTIONS class << self - def defaults(key) + def defaults(key, server_type) if key && self::DEFAULTS.key?(key.to_sym) - Git::DEFAULTS.merge self::DEFAULTS[key.to_sym] + Travis::Vcs.defaults(server_type).merge self::DEFAULTS[key.to_sym] elsif self::DEFAULTS[:default] - Git::DEFAULTS.merge self::DEFAULTS[:default] + Travis::Vcs.defaults(server_type).merge self::DEFAULTS[:default] else - Git::DEFAULTS.merge self::DEFAULTS + Travis::Vcs.defaults(server_type).merge self::DEFAULTS end end end @@ -111,9 +111,10 @@ def defaults(key) def initialize(data) @raw_data = data.deep_symbolize_keys raw_config = @raw_data[:config] + server_type = @raw_data.dig(:repository, :server_type) || 'git' lang_sym = raw_config.fetch(:language,"").to_sym @data = Data.new({ - config: self.class.defaults(raw_config[:os]), + config: self.class.defaults(raw_config[:os], server_type), language_default_p: !raw_config[lang_sym] }.deep_merge(self.raw_data)) @options = {} @@ -350,6 +351,8 @@ def configure end def setup_filter + apply :vault_connect + apply :vault_keys apply :no_world_writable_dirs apply :setup_filter end diff --git a/lib/travis/build/script/clojure.rb b/lib/travis/build/script/clojure.rb index df8e1d8dd3..b1f5c08150 100644 --- a/lib/travis/build/script/clojure.rb +++ b/lib/travis/build/script/clojure.rb @@ -54,6 +54,7 @@ def update_lein(version) sh.echo "Updating leiningen to #{version}", ansi: :yellow sh.cmd "env LEIN_ROOT=true curl -L -o /usr/local/bin/lein https://raw.githubusercontent.com/technomancy/leiningen/#{version}/bin/lein", echo: true, assert: true, sudo: true sh.cmd "rm -rf ${TRAVIS_HOME}/.lein", echo: false + sh.chmod "+x", "/usr/local/bin/lein", sudo: true sh.cmd "lein self-install", echo: true, assert: true end end diff --git a/lib/travis/build/script/shared/rvm.rb b/lib/travis/build/script/shared/rvm.rb index 3e4b76964b..0e6223a7ae 100644 --- a/lib/travis/build/script/shared/rvm.rb +++ b/lib/travis/build/script/shared/rvm.rb @@ -122,7 +122,12 @@ def use_default_ruby def use_ruby_version_file sh.fold('rvm') do sh.if '-n $(grep "^3" .ruby-version)' do - sh.cmd 'rvm get head' + sh.if '$(uname) = "Darwin"' do + sh.echo "Installing Ruby (skipping rvm update)", ansi: :yellow + end + sh.else do + sh.cmd 'rvm get head' + end end sh.cmd 'rvm use $(< .ruby-version) --install --binary --fuzzy' end @@ -160,7 +165,12 @@ def use_ruby_version end else if ruby_version.start_with? '3' - sh.cmd "rvm get head" + sh.if '$(uname) = "Darwin"' do + sh.echo "Installing Ruby (skipping rvm update)", ansi: :yellow + end + sh.else do + sh.cmd "rvm get head" + end end sh.cmd "rvm use #{ruby_version} --install --binary --fuzzy" end diff --git a/lib/travis/services/vault.rb b/lib/travis/services/vault.rb new file mode 100644 index 0000000000..7d223c9dd8 --- /dev/null +++ b/lib/travis/services/vault.rb @@ -0,0 +1,9 @@ +require 'travis/services/vault/keys' +require 'travis/services/vault/connect' + +module Travis + module Vault + class ConnectionError < StandardError; end + class RootKeyError < StandardError; end + end +end diff --git a/lib/travis/services/vault/connect.rb b/lib/travis/services/vault/connect.rb new file mode 100644 index 0000000000..2e5d86a114 --- /dev/null +++ b/lib/travis/services/vault/connect.rb @@ -0,0 +1,14 @@ +require 'rest-client' + +module Travis + module Vault + class Connect + def self.call(vault) + response = RestClient.get("#{vault[:api_url]}/v1/auth/token/lookup-self", 'X-Vault-Token': vault[:token]) + raise ConnectionError if response.code != 200 + rescue RestClient::ExceptionWithResponse, SocketError + raise ConnectionError + end + end + end +end diff --git a/lib/travis/services/vault/keys.rb b/lib/travis/services/vault/keys.rb new file mode 100644 index 0000000000..1b78b22314 --- /dev/null +++ b/lib/travis/services/vault/keys.rb @@ -0,0 +1,25 @@ +require 'travis/services/vault/keys/kv1' +require 'travis/services/vault/keys/kv2' + +require 'travis/services/vault/keys/paths' +require 'travis/services/vault/keys/version' +require 'travis/services/vault/keys/resolver' + +module Travis + module Vault + class Keys + + attr_reader :appliance + + def initialize(appliance) + @appliance = appliance + end + + def resolve + paths = Paths.call(appliance.vault) + version = Version.call(appliance.vault) + Resolver.new(paths, version, appliance).call + end + end + end +end diff --git a/lib/travis/services/vault/keys/build_paths.rb b/lib/travis/services/vault/keys/build_paths.rb new file mode 100644 index 0000000000..fb3316be1d --- /dev/null +++ b/lib/travis/services/vault/keys/build_paths.rb @@ -0,0 +1,35 @@ +module Travis + module Vault + class Keys + class BuildPaths + + attr_reader :secrets + + def initialize(secrets) + @secrets = secrets + end + + def call + secrets.map { |secret| format_paths(secret) } + .flatten + .reverse + .uniq + .reverse + end + + private + + def format_paths(secret) + return secret if secret.is_a?(String) + return [] if secret[:namespace].blank? + namespace_name = secret[:namespace].find { |el| el.is_a?(Hash) && el&.dig(:name) }&.dig(:name) + + return secret[:namespace] if namespace_name.blank? + + paths = secret[:namespace].reject { |el| el.is_a?(Hash) && el[:name] } + paths.map { |path| "#{namespace_name}/#{path}" } + end + end + end + end +end diff --git a/lib/travis/services/vault/keys/kv1.rb b/lib/travis/services/vault/keys/kv1.rb new file mode 100644 index 0000000000..2c2be2c257 --- /dev/null +++ b/lib/travis/services/vault/keys/kv1.rb @@ -0,0 +1,16 @@ +require 'rest-client' + +module Travis + module Vault + class Keys + class KV1 + def self.resolve(path, vault) + response = RestClient.get("#{vault[:api_url]}/v1/secret/#{path}", 'X-Vault-Token': vault[:token]) + JSON.parse(response.body)['data'] if response.code == 200 + rescue RestClient::ExceptionWithResponse, SocketError + nil + end + end + end + end +end diff --git a/lib/travis/services/vault/keys/kv2.rb b/lib/travis/services/vault/keys/kv2.rb new file mode 100644 index 0000000000..5a840ba70e --- /dev/null +++ b/lib/travis/services/vault/keys/kv2.rb @@ -0,0 +1,16 @@ +require 'rest-client' + +module Travis + module Vault + class Keys + class KV2 + def self.resolve(path, vault) + response = RestClient.get("#{vault[:api_url]}/v1/secret/data/#{path}", 'X-Vault-Token': vault[:token]) + JSON.parse(response.body).dig('data', 'data') if response.code == 200 + rescue RestClient::ExceptionWithResponse, SocketError + nil + end + end + end + end +end diff --git a/lib/travis/services/vault/keys/paths.rb b/lib/travis/services/vault/keys/paths.rb new file mode 100644 index 0000000000..6bff62e7a9 --- /dev/null +++ b/lib/travis/services/vault/keys/paths.rb @@ -0,0 +1,15 @@ +require 'travis/services/vault/keys/build_paths' + +module Travis + module Vault + class Keys + class Paths + def self.call(vault) + paths = vault[:secrets].reject { |secret| secret.is_a?(Hash) && secret[:kv_api_ver] } + + BuildPaths.new(paths).call + end + end + end + end +end diff --git a/lib/travis/services/vault/keys/resolver.rb b/lib/travis/services/vault/keys/resolver.rb new file mode 100644 index 0000000000..bb0c82e1a0 --- /dev/null +++ b/lib/travis/services/vault/keys/resolver.rb @@ -0,0 +1,49 @@ +module Travis + module Vault + class Keys + class Resolver + attr_reader :paths, :version, :appliance + + delegate :data, to: :appliance + delegate :vault, to: :appliance + delegate :export, :echo, to: :'appliance.sh' + + def initialize(paths, version, appliance) + @paths = paths + @version = version + @appliance = appliance + end + + def call + return if paths.blank? + + vault_secrets = [] + + paths.each do |path| + secret_data = Keys.const_get(version.upcase).resolve(path, vault) + if secret_data.present? + secret_name = path.split('/').last + secret_data.each do |key, value| + env_name = key + env_name = [secret_name, env_name].join('_') if true # To-Do: Make the prepend customizable from .travis.yml + env_name = (path.split('/') << env_name).join('_') if false # To-Do: Make the prepend customizable from .travis.yml + export(env_name.upcase, %("#{value}"), echo: false, secure: true) + vault_secrets << value + end + else + echo *(warn_message(path)) + end + end + + data.vault_secrets = vault_secrets.uniq if vault_secrets.present? + end + + private + + def warn_message(path) + ["The value fetched for #{path} is blank.", ansi: :yellow] + end + end + end + end +end diff --git a/lib/travis/services/vault/keys/version.rb b/lib/travis/services/vault/keys/version.rb new file mode 100644 index 0000000000..e84bef0cf2 --- /dev/null +++ b/lib/travis/services/vault/keys/version.rb @@ -0,0 +1,13 @@ +module Travis + module Vault + class Keys + class Version + DEFAULT_VALUE = 'kv2'.freeze + + def self.call(vault) + vault[:secrets].find { |secret| secret.is_a?(Hash) && secret[:kv_api_ver] }&.values&.first || DEFAULT_VALUE + end + end + end + end +end diff --git a/lib/travis/vcs.rb b/lib/travis/vcs.rb new file mode 100644 index 0000000000..31ed44c425 --- /dev/null +++ b/lib/travis/vcs.rb @@ -0,0 +1,72 @@ +require 'travis/vcs/base.rb' +require 'travis/vcs/git.rb' +require 'travis/vcs/perforce.rb' +require 'travis/vcs/svn.rb' +require 'travis/build/errors.rb' + +module Travis + module Vcs + class </dev/null`.strip + ) + end + + def self.version + @version ||= `git describe --always --dirty --tags 2>/dev/null`.strip + end + + def self.paths + @paths ||= '$(git ls-files -o | tr "\n" ":")' + end - attr_reader :sh, :data + def self.clone_cmd(endpoint, source) + "git clone #{endpoint}/#{source} #{source}" + end + + def self.checkout_cmd(branch) + "git checkout #{branch}" + end - def initialize(sh, data) - @sh = sh - @data = data + def self.revision_cmd + @rev ||= 'git rev-parse HEAD' end + def self.defaults + DEFAULTS + end + + def checkout disable_interactive_auth enable_longpaths if config[:os] == 'windows' diff --git a/lib/travis/build/git/clone.rb b/lib/travis/vcs/git/clone.rb similarity index 99% rename from lib/travis/build/git/clone.rb rename to lib/travis/vcs/git/clone.rb index 0b2a716024..870cb480e8 100644 --- a/lib/travis/build/git/clone.rb +++ b/lib/travis/vcs/git/clone.rb @@ -1,9 +1,9 @@ require 'shellwords' -require 'travis/build/git/netrc' +require 'travis/vcs/git/netrc' module Travis - module Build - class Git + module Vcs + class Git < Base class Clone < Struct.new(:sh, :data) def apply sh.fold 'git.checkout' do diff --git a/lib/travis/build/git/netrc.rb b/lib/travis/vcs/git/netrc.rb similarity index 96% rename from lib/travis/build/git/netrc.rb rename to lib/travis/vcs/git/netrc.rb index cb1dab835c..7a53678188 100644 --- a/lib/travis/build/git/netrc.rb +++ b/lib/travis/vcs/git/netrc.rb @@ -1,6 +1,6 @@ module Travis - module Build - class Git + module Vcs + class Git < Base class Netrc < Struct.new(:sh, :data) def apply sh.echo "Using ${TRAVIS_HOME}/#{netrc_filename} to clone repository." diff --git a/lib/travis/build/git/ssh_key.rb b/lib/travis/vcs/git/ssh_key.rb similarity index 97% rename from lib/travis/build/git/ssh_key.rb rename to lib/travis/vcs/git/ssh_key.rb index dbba9a3e36..1941f962bc 100644 --- a/lib/travis/build/git/ssh_key.rb +++ b/lib/travis/vcs/git/ssh_key.rb @@ -1,6 +1,6 @@ module Travis - module Build - class Git + module Vcs + class Git < Base class SshKey < Struct.new(:sh, :data) def apply sh.fold 'ssh_key' do diff --git a/lib/travis/build/git/submodules.rb b/lib/travis/vcs/git/submodules.rb similarity index 95% rename from lib/travis/build/git/submodules.rb rename to lib/travis/vcs/git/submodules.rb index 4595119b03..c56e25ca62 100644 --- a/lib/travis/build/git/submodules.rb +++ b/lib/travis/vcs/git/submodules.rb @@ -1,8 +1,8 @@ require 'shellwords' module Travis - module Build - class Git + module Vcs + class Git < Base class Submodules < Struct.new(:sh, :data) def apply sh.if '-f .gitmodules' do diff --git a/lib/travis/build/git/tarball.rb b/lib/travis/vcs/git/tarball.rb similarity index 97% rename from lib/travis/build/git/tarball.rb rename to lib/travis/vcs/git/tarball.rb index 9ce637866b..8d505720bf 100644 --- a/lib/travis/build/git/tarball.rb +++ b/lib/travis/vcs/git/tarball.rb @@ -1,6 +1,6 @@ module Travis - module Build - class Git + module Vcs + class Git < Base class Tarball < Struct.new(:sh, :data) def apply sh.fold 'git.tarball' do diff --git a/lib/travis/vcs/perforce.rb b/lib/travis/vcs/perforce.rb new file mode 100644 index 0000000000..57c822a2aa --- /dev/null +++ b/lib/travis/vcs/perforce.rb @@ -0,0 +1,149 @@ +require 'travis/vcs/base' +require 'travis/vcs/perforce/clone' +require 'travis/vcs/perforce/netrc' +require 'travis/vcs/perforce/ssh_key' +require 'travis/vcs/perforce/submodules' +require 'travis/vcs/perforce/tarball' +module Travis + module Vcs + class Perforce < Base + DEFAULTS = { + perforce: { + depth: 50, + submodules: true, + strategy: 'clone', + quiet: false, + lfs_skip_smudge: false, + sparse_checkout: false, + clone: true + } + } + def self.top + @top ||= Pathname.new( + `p4 info | grep 'Client root:' | cut -d ' ' -f 3- 2>/dev/null`.strip + ) + end + + def self.version + @version ||= `p4 changes -m1 2>/dev/null`.strip + end + + def self.paths + @paths ||= '$(git ls-files -o | tr "\n" ":")' + end + + def self.clone_cmd(endpoint, source) + "p4 -p #{host} -d #{source} clone -f #{endpoint}/#{source}" + end + + def self.checkout_cmd(branch) + "p4 switch #{branch}" + end + + def self.revision_cmd + @rev ||= 'p4 changes -m1' + end + + def self.defaults + DEFAULTS + end + + def checkout + disable_interactive_auth + enable_longpaths if config[:os] == 'windows' + install_ssh_key if install_ssh_key? + write_netrc if write_netrc? + sh.newline + + if use_tarball? + download_tarball + else + config_symlink + clone_or_fetch + submodules + end + delete_netrc if delete_netrc? + end + + private + + def disable_interactive_auth + #TODO ? + end + + def enable_longpaths + #TODO ? + end + + def install_ssh_key? + data.ssh_key? + end + + def netrc + @netrc ||= Netrc.new(sh, data) + end + + def write_netrc? + data.installation? && !data.custom_ssh_key? or data.prefer_https? + end + + def write_netrc + netrc.apply + end + + def delete_netrc? + !data.keep_netrc? + end + + def delete_netrc + netrc.delete + end + + def install_ssh_key + SshKey.new(sh, data).apply + end + + def download_tarball + Tarball.new(sh, data).apply + end + + def clone_or_fetch + if clone? + Clone.new(sh, data).apply + else + sh.echo 'Skipping \`git clone\` based on given configuration', ansi: :yellow + end + end + + def submodules + Submodules.new(sh, data).apply if submodules? + end + + def config + DEFAULTS.merge(data.config) + end + + def submodules? + config[:perforce][:submodules] + end + + def use_tarball? + config[:perforce][:strategy] == 'tarball' + end + + def clone? + config[:perforce][:clone] + end + + def config_symlink + if config[:perforce].key? :symlinks + sh.cmd "git config --global core.symlinks #{!!config[:perforce][:symlinks]}", echo: false, assert: false, timing: false + end + end + + def dir + data.slug + end + end + end +end diff --git a/lib/travis/vcs/perforce/clone.rb b/lib/travis/vcs/perforce/clone.rb new file mode 100644 index 0000000000..8048a12bb1 --- /dev/null +++ b/lib/travis/vcs/perforce/clone.rb @@ -0,0 +1,125 @@ +require 'shellwords' +require 'travis/vcs/perforce/netrc' + +module Travis + module Vcs + class Perforce < Base + class Clone < Struct.new(:sh, :data) + def apply + sh.fold 'p4.checkout' do + clone + sh.cd 'tempdir' + checkout + end + sh.newline + end + + private + DEFAULT_TRACE_COMMAND = '' + + def repo_slug + data.repository[:slug].to_s + end + + def owner_login + repo_slug.split('/').first + end + + def trace_p4_commands? + false + end + + def trace_command + DEFAULT_TRACE_COMMAND + end + + def clone + sh.export 'P4USER', user, echo: true, assert: false + sh.export 'P4CHARSET', 'utf8', echo: false, assert: false + sh.export 'P4PASSWD', ticket, echo: false, assert: false + sh.export 'P4PORT', port, echo: false, assert: false + sh.cmd 'p4 trust -y' + sh.cmd "p4 #{p4_opt} client -S //#{dir}/#{checkout_ref} -o | p4 #{p4_opt} client -i" + sh.cmd "p4 #{p4_opt} sync -p" + end + + def p4_opt + '-v ssl.client.trust.name=1' + end + + def checkout + #sh.cmd "p4 -c tempdir switch #{checkout_ref}", timing: false + end + + def checkout_ref + return branch if data.branch + return tag if data.tag + data.commit + end + + def clone_args + args = " -p #{host}" + args << " -r #{remote}" + args << " -v" if trace_p4_commands? + args + end + + def autocrlf_key_given? + config[:perforce].key?(:autocrlf) + end + + def host + config[:perforce].host + end + + def remote + config[:perforce].remote + end + + def branch + data.branch.shellescape if data.branch + end + + def tag + data.tag.shellescape if data.tag + end + + def quiet? + config[:perforce][:quiet] + end + + def lfs_skip_smudge? + config[:perforce][:lfs_skip_smudge] == true + end + + def sparse_checkout + config[:perforce][:sparse_checkout] + end + + def dir + assembla? ? 'depot' : data.slug.split('/').last + end + + def port + data[:repository][:source_url]&.split('/').first + end + + def user + data[:sender_login] + end + + def ticket + data[:build_token] + end + + def config + data.config + end + + def assembla? + @assembla ||= data[:repository][:source_url].include? 'assembla' + end + end + end + end +end diff --git a/lib/travis/vcs/perforce/netrc.rb b/lib/travis/vcs/perforce/netrc.rb new file mode 100644 index 0000000000..d1f6b8473f --- /dev/null +++ b/lib/travis/vcs/perforce/netrc.rb @@ -0,0 +1,31 @@ +module Travis + module Vcs + class Perforce < Base + class Netrc < Struct.new(:sh, :data) + def apply + sh.echo "Using ${TRAVIS_HOME}/#{netrc_filename} to clone repository." + sh.raw "echo -e #{Shellwords.escape netrc_content} > ${TRAVIS_HOME}/#{netrc_filename}" + sh.raw "chmod 0600 ${TRAVIS_HOME}/#{netrc_filename}" + end + + def delete + sh.raw "rm -f ${TRAVIS_HOME}/#{netrc_filename}" + end + + private + + def netrc_content + if data.installation? + "machine #{data.source_host}\n login travis-ci\n password #{data.token}\n" + else + "machine #{data.source_host}\n login #{data.token}\n" + end + end + + def netrc_filename + data.config[:os].to_s.downcase == 'windows' ? '_netrc' : '.netrc' + end + end + end + end +end diff --git a/lib/travis/vcs/perforce/ssh_key.rb b/lib/travis/vcs/perforce/ssh_key.rb new file mode 100644 index 0000000000..86d4817218 --- /dev/null +++ b/lib/travis/vcs/perforce/ssh_key.rb @@ -0,0 +1,39 @@ +module Travis + module Vcs + class Perforce < Base + class SshKey < Struct.new(:sh, :data) + def apply + sh.fold 'ssh_key' do + sh.echo messages + end + + sh.mkdir '~/.ssh', recursive: true, echo: false + sh.file '~/.ssh/id_rsa', key.value + sh.chmod 600, '~/.ssh/id_rsa', echo: false + sh.raw 'eval `ssh-agent` &> /dev/null' + sh.raw 'ssh-add ~/.ssh/id_rsa &> /dev/null' + + # BatchMode - If set to 'yes', passphrase/password querying will be disabled. + # TODO ... how to solve StrictHostKeyChecking correctly? deploy a known_hosts file? + sh.file '~/.ssh/config', "Host #{data.source_host}\n\tBatchMode yes\n\tStrictHostKeyChecking no\n", append: true + end + + private + + def key + data.ssh_key + end + + def messages + msgs = ["Installing SSH key#{" from: #{source}" if key.source}"] + msgs << "Key fingerprint: #{key.fingerprint}" if key.fingerprint + msgs + end + + def source + key.source.gsub(/[_-]+/, ' ') + end + end + end + end +end diff --git a/lib/travis/vcs/perforce/submodules.rb b/lib/travis/vcs/perforce/submodules.rb new file mode 100644 index 0000000000..4bd210bcbd --- /dev/null +++ b/lib/travis/vcs/perforce/submodules.rb @@ -0,0 +1,18 @@ +require 'shellwords' + +module Travis + module Vcs + class Perforce < Base + class Submodules < Struct.new(:sh, :data) + def apply + end + + private + + def config + data.config + end + end + end + end +end diff --git a/lib/travis/vcs/perforce/tarball.rb b/lib/travis/vcs/perforce/tarball.rb new file mode 100644 index 0000000000..8e46090be4 --- /dev/null +++ b/lib/travis/vcs/perforce/tarball.rb @@ -0,0 +1,57 @@ +module Travis + module Vcs + class Perforce < Base + class Tarball < Struct.new(:sh, :data) + def apply + sh.fold 'p4.tarball' do + mkdir + download + extract + move + end + end + + private + + def mkdir + sh.mkdir dir, echo: false, recursive: true + end + + def download + cmd = "curl -o #{filename} #{auth_header}-L #{tarball_url}" + echo = cmd.gsub(data.token || /\Za/, '[SECURE]') + sh.cmd cmd, echo: echo, retry: true + end + + def extract + sh.cmd "tar xfz #{filename}" + end + + def move + sh.mv "#{basename}-#{data.commit[0..6]}/*", dir, echo: false + sh.cd dir + end + + def dir + data.slug + end + + def filename + "#{basename}.tar.gz" + end + + def basename + data.slug.gsub('/', '-') + end + + def tarball_url + "#{data.api_url}/tarball/#{data.commit}" + end + + def auth_header + "-H \"Authorization: token #{data.token}\" " if data.token + end + end + end + end +end diff --git a/lib/travis/vcs/svn.rb b/lib/travis/vcs/svn.rb new file mode 100644 index 0000000000..743f4a32ae --- /dev/null +++ b/lib/travis/vcs/svn.rb @@ -0,0 +1,149 @@ +require 'travis/vcs/base' +require 'travis/vcs/svn/clone' +require 'travis/vcs/svn/netrc' +require 'travis/vcs/svn/ssh_key' +require 'travis/vcs/svn/submodules' +require 'travis/vcs/svn/tarball' +module Travis + module Vcs + class Svn < Base + DEFAULTS = { + svn: { + depth: 50, + submodules: true, + strategy: 'clone', + quiet: false, + lfs_skip_smudge: false, + sparse_checkout: false, + clone: true + } + } + def self.top + @top ||= Pathname.new( + `svn info --show-item wc-root 2>/dev/null`.strip + ) + end + + def self.version + @version ||= `svn info -r HEAD --show-item last-changed-revision 2>/dev/null`.strip + end + + def self.paths + @paths ||= '$(svn status 2>/dev/null | grep ^? | cut -d ' ' -f 8- |tr "\n" ":")' + end + + def self.clone_cmd(endpoint, source) + "svn co #{endpoint}/#{source} #{source}" + end + + def self.checkout_cmd(branch) + "svn sw ^/branches/#{branch}" + end + + def self.revision_cmd + @rev ||= 'svn info -r HEAD --show-item last-changed-revision 2>/dev/null' + end + + def self.defaults + DEFAULTS + end + + def checkout + disable_interactive_auth + enable_longpaths if config[:os] == 'windows' + install_ssh_key if install_ssh_key? + write_netrc if write_netrc? + sh.newline + + if use_tarball? + download_tarball + else + config_symlink + clone_or_fetch + submodules + end + delete_netrc if delete_netrc? + end + + private + + def disable_interactive_auth + #TODO ? + end + + def enable_longpaths + #TODO ? + end + + def install_ssh_key? + data.ssh_key? + end + + def netrc + @netrc ||= Netrc.new(sh, data) + end + + def write_netrc? + data.installation? && !data.custom_ssh_key? or data.prefer_https? + end + + def write_netrc + netrc.apply + end + + def delete_netrc? + !data.keep_netrc? + end + + def delete_netrc + netrc.delete + end + + def install_ssh_key + SshKey.new(sh, data).apply + end + + def download_tarball + Tarball.new(sh, data).apply + end + + def clone_or_fetch + if clone? + Clone.new(sh, data).apply + else + sh.echo 'Skipping \`git clone\` based on given configuration', ansi: :yellow + end + end + + def submodules + Submodules.new(sh, data).apply if submodules? + end + + def config + DEFAULTS.merge(data.config) + end + + def submodules? + config[:svn][:submodules] + end + + def use_tarball? + config[:svn][:strategy] == 'tarball' + end + + def clone? + config[:svn][:clone] + end + + def config_symlink + if config[:svn].key? :symlinks + sh.cmd "git config --global core.symlinks #{!!config[:svn][:symlinks]}", echo: false, assert: false, timing: false + end + end + + def dir + data.slug + end + end + end +end diff --git a/lib/travis/vcs/svn/clone.rb b/lib/travis/vcs/svn/clone.rb new file mode 100644 index 0000000000..c1d7b57a16 --- /dev/null +++ b/lib/travis/vcs/svn/clone.rb @@ -0,0 +1,105 @@ +require 'shellwords' +require 'travis/vcs/svn/netrc' + +module Travis + module Vcs + class Svn < Base + class Clone < Struct.new(:sh, :data) + def apply + sh.fold 'svn.checkout' do + clone + sh.cd repository_name + checkout + end + sh.newline + end + + private + DEFAULT_TRACE_COMMAND = '' + + def repo_slug + data.repository[:slug].to_s + end + + def source_url + if assembla? + return "svn+ssh://#{data.repository[:source_host]}" unless data.repository[:source_host].start_with?('svn+ssh://') + + return data.repository[:source_host] + end + + data.repository[:source_url] + end + + def source_host + data.repository[:source_host] + end + + def assembla? + @assembla ||= source_host.include? 'assembla' + end + + def owner_login + repo_slug.split('/').first + end + + def trace_command + DEFAULT_TRACE_COMMAND + end + + def clone + sh.cmd "svn co #{source_url}#{clone_args} #{repository_name}", assert: false, retry: true + end + + def checkout + sh.cmd "svn update -r #{checkout_ref}", timing: false + end + + def checkout_ref + ref = if data.tag + tag + else + data.commit + end + ref = ref.split('@')[1] if ref.include?('@') + + ref + end + + def clone_args + args = "" + if branch && branch == 'trunk' + args << "/#{branch}" + else + args << "/branches/#{branch}" if branch + end + args + end + + def host + URI(source_host)&.host + end + + def repository_name + repo_slug&.split('/').last + end + + def branch + data.branch.shellescape if data.branch + end + + def tag + data.tag.shellescape if data.tag + end + + def user + data[:sender_login] + end + + def config + data.config + end + end + end + end +end diff --git a/lib/travis/vcs/svn/netrc.rb b/lib/travis/vcs/svn/netrc.rb new file mode 100644 index 0000000000..130f04d983 --- /dev/null +++ b/lib/travis/vcs/svn/netrc.rb @@ -0,0 +1,31 @@ +module Travis + module Vcs + class Svn < Base + class Netrc < Struct.new(:sh, :data) + def apply + sh.echo "Using ${TRAVIS_HOME}/#{netrc_filename} to clone repository." + sh.raw "echo -e #{Shellwords.escape netrc_content} > ${TRAVIS_HOME}/#{netrc_filename}" + sh.raw "chmod 0600 ${TRAVIS_HOME}/#{netrc_filename}" + end + + def delete + sh.raw "rm -f ${TRAVIS_HOME}/#{netrc_filename}" + end + + private + + def netrc_content + if data.installation? + "machine #{data.source_host}\n login travis-ci\n password #{data.token}\n" + else + "machine #{data.source_host}\n login #{data.token}\n" + end + end + + def netrc_filename + data.config[:os].to_s.downcase == 'windows' ? '_netrc' : '.netrc' + end + end + end + end +end diff --git a/lib/travis/vcs/svn/ssh_key.rb b/lib/travis/vcs/svn/ssh_key.rb new file mode 100644 index 0000000000..d1a7a2076b --- /dev/null +++ b/lib/travis/vcs/svn/ssh_key.rb @@ -0,0 +1,45 @@ +module Travis + module Vcs + class Svn < Base + class SshKey < Struct.new(:sh, :data) + def apply + sh.mkdir '~/.ssh', recursive: true, echo: false + sh.file '~/.ssh/id_rsa', key + sh.chmod 600, '~/.ssh/id_rsa', echo: false + sh.raw 'eval `ssh-agent` &> /dev/null' + sh.raw 'ssh-add ~/.ssh/id_rsa &> /dev/null' + + # BatchMode - If set to 'yes', passphrase/password querying will be disabled. + # TODO ... how to solve StrictHostKeyChecking correctly? deploy a known_hosts file? + sh.file '~/.ssh/config', "Host #{source_host}\n\tBatchMode yes\n\tStrictHostKeyChecking no\n\tSendEnv REPO_NAME", append: true + sh.export 'REPO_NAME', repository_name, echo: false + + #TODO why it's not reading the config + sh.export 'SVN_SSH', '"ssh -o SendEnv=REPO_NAME -o StrictHostKeyChecking=no -l svn"', echo: false if assembla? + end + + private + + def key + data[:build_token] + end + + def repository_name + repo_slug&.split('/').last + end + + def repo_slug + data.repository[:slug].to_s + end + + def source_host + data.repository[:source_host] + end + + def assembla? + @assembla ||= source_host.include? 'assembla' + end + end + end + end +end diff --git a/lib/travis/vcs/svn/submodules.rb b/lib/travis/vcs/svn/submodules.rb new file mode 100644 index 0000000000..57e76d9281 --- /dev/null +++ b/lib/travis/vcs/svn/submodules.rb @@ -0,0 +1,18 @@ +require 'shellwords' + +module Travis + module Vcs + class Svn < Base + class Submodules < Struct.new(:sh, :data) + def apply + end + + private + + def config + data.config + end + end + end + end +end diff --git a/lib/travis/vcs/svn/tarball.rb b/lib/travis/vcs/svn/tarball.rb new file mode 100644 index 0000000000..1e8cbc3b9c --- /dev/null +++ b/lib/travis/vcs/svn/tarball.rb @@ -0,0 +1,57 @@ +module Travis + module Vcs + class Svn < Base + class Tarball < Struct.new(:sh, :data) + def apply + sh.fold 'svn.tarball' do + mkdir + download + extract + move + end + end + + private + + def mkdir + sh.mkdir dir, echo: false, recursive: true + end + + def download + cmd = "curl -o #{filename} #{auth_header}-L #{tarball_url}" + echo = cmd.gsub(data.token || /\Za/, '[SECURE]') + sh.cmd cmd, echo: echo, retry: true + end + + def extract + sh.cmd "tar xfz #{filename}" + end + + def move + sh.mv "#{basename}-#{data.commit[0..6]}/*", dir, echo: false + sh.cd dir + end + + def dir + data.slug + end + + def filename + "#{basename}.tar.gz" + end + + def basename + data.slug.gsub('/', '-') + end + + def tarball_url + "#{data.api_url}/tarball/#{data.commit}" + end + + def auth_header + "-H \"Authorization: token #{data.token}\" " if data.token + end + end + end + end +end diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index aae8379fb2..da5d0b2239 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,24 +166,26 @@ "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.4.2", - "9.x": "9.4.2", - "9.x.x": "9.4.2", + "9": "9.4.3", + "9.x": "9.4.3", + "9.x.x": "9.4.3", "9.0.x": "9.0.2", "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", - "9.2": "9.2.4", + "9.2": "9.2.5", "9.2.1-alpha2": "9.2.1-alpha2", "9.2.1": "9.2.1", - "9.2.x": "9.2.4", + "9.2.x": "9.2.5", "9.2.2": "9.2.2", "9.2.3": "9.2.3", "9.2.4": "9.2.4", + "9.2.5": "9.2.5", "9.4.1-alpha1": "9.4.1-alpha1", - "9.4": "9.4.2", + "9.4": "9.4.3", "9.4.1-alpha2": "9.4.1-alpha2", "9.4.1-alpha3": "9.4.1-alpha3", "9.4.1": "9.4.1", - "9.4.x": "9.4.2", - "9.4.2": "9.4.2" + "9.4.x": "9.4.3", + "9.4.2": "9.4.2", + "9.4.3": "9.4.3" } \ No newline at end of file diff --git a/spec/build/appliances/vault_connect_spec.rb b/spec/build/appliances/vault_connect_spec.rb new file mode 100644 index 0000000000..d957f895f3 --- /dev/null +++ b/spec/build/appliances/vault_connect_spec.rb @@ -0,0 +1,130 @@ +require 'spec_helper' + +describe Travis::Build::Appliances::VaultConnect do + let(:instance) { described_class.new } + + describe '#apply?' do + subject(:apply?) { instance.apply? } + + context 'real world problem' do + before do + config = { + vault: { + api_url: 'some/url', + token: { + secure: 'aZ38cdlu8w/kzWHFv8P/wFFJgCbfGxROpjh4J3OcUUaAaZkuqD9Y7WdyoU2Ofu3d8byUPxRkElWHD6BI=' + }, + secrets: [ + { namespace: [ + { name: 'ns1' }, + 'project_id/secret_key_a', + 'project_id/not_existent', + ] }, + 'ns2/project_id/secret_key_b', + 'ns2/project_id/not_existent_key' + ] + } + } + instance.stubs(:config).returns(config) + end + + it 'returns truthy value' do + is_expected.to be_truthy + end + end + + context 'when there is a vault in a config' do + before do + instance.stubs(:config).returns(vault: { secrets: %w[aaa/bbb] }) + end + + it 'sets @vault variable to vault value' do + apply? + + expect(instance.instance_variable_get(:@vault)).to eq({ secrets: %w[aaa/bbb] }) + end + + it 'returns truthy value' do + is_expected.to be_truthy + end + end + + shared_examples "it won't connect to the vault for given config" do |config| + before do + instance.stubs(:config).returns(config) + end + + it 'sets @vault variable nil' do + apply? + + expect(instance.instance_variable_get(:@vault)).to eq(nil) + end + + it 'returns falsey value' do + is_expected.to be_falsey + end + end + + include_examples "it won't connect to the vault for given config", { vault: { secrets: [] } } + include_examples "it won't connect to the vault for given config", { vault: { secrets: [{ kv_api_ver: 'kv2' }] } } + include_examples "it won't connect to the vault for given config", { vault: + { secrets: + [ + { kv_api_ver: 'kv2' }, + { namespace: { name: 'blah' } } + ] + } + } + end + + describe 'apply' do + subject(:apply) { instance.apply } + + before do + instance.instance_variable_set(:@vault, { token: 'my_token', api_url: 'https://api_url.com' }) + end + + describe 'connection to the vault' do + let(:sh) do + stub('sh') + end + + context 'when it is proper' do + before do + Travis::Vault::Connect.stubs(:call) + instance.stubs(:sh).returns(sh) + end + + it 'writes the success message, export vault config variables in the console and does not terminates the job' do + Travis::Vault::Connect.expects(:call).with({ token: 'my_token', api_url: 'https://api_url.com' }) + sh.expects(:echo).with('Connected to Vault instance.', ansi: :green) + sh.expects(:export).with('VAULT_ADDR', 'https://api_url.com', echo: true, secure: true) + sh.expects(:export).with('VAULT_TOKEN', 'my_token', echo: true, secure: true) + sh.expects(:terminate).never + + apply + end + end + + context 'when it is not' do + shared_examples 'it terminates a job with a message for' do |error_class| + before do + Travis::Vault::Connect.stubs(:call).raises(error_class) + instance.stubs(:sh).returns(sh) + end + + it 'writes the error message in the console and terminates the job' do + sh.expects(:echo).with("Failed to connect to the Vault instance. Please verify if:\n* The Vault Token is correct (encrypted, not plain text). \n* The Vault Token is not expired. \n* The Vault can accept connections from the Travis CI build job environments (https://docs.travis-ci.com/user/ip-addresses/).", ansi: :red) + sh.expects(:terminate) + + apply + end + end + + include_examples 'it terminates a job with a message for', Travis::Vault::ConnectionError + include_examples 'it terminates a job with a message for', ArgumentError + include_examples 'it terminates a job with a message for', URI::InvalidURIError + end + end + end +end diff --git a/spec/build/appliances/vault_keys_spec.rb b/spec/build/appliances/vault_keys_spec.rb new file mode 100644 index 0000000000..c721f5a74f --- /dev/null +++ b/spec/build/appliances/vault_keys_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +describe Travis::Build::Appliances::VaultKeys do + let(:instance) { described_class.new } + + describe '#vault' do + it do + expect(instance.respond_to?(:vault)).to be(true) + end + end + + describe '#apply?' do + subject(:apply?) { instance.apply? } + + context 'when there is a vault in a config' do + before do + instance.stubs(:config).returns(vault: { secrets: %w[aaa/bbb] }) + end + + it 'sets @vault variable to vault value' do + apply? + + expect(instance.instance_variable_get(:@vault)).to eq({ secrets: %w[aaa/bbb] }) + end + + it 'returns truthy value' do + is_expected.to be_truthy + end + end + + context 'when there is no a vault in a config' do + before do + instance.stubs(:config).returns({ secrets: [] }) + end + + it 'sets @vault variable nil' do + apply? + + expect(instance.instance_variable_get(:@vault)).to eq(nil) + end + + it 'returns falsey value' do + is_expected.to be_falsey + end + end + end + + describe '#apply' do + subject(:apply) { instance.apply } + + let(:vault_keys) { stub(:vault_keys) } + + context 'a normal scenario' do + it do + Travis::Vault::Keys.expects(:new).with(instance).returns(vault_keys) + vault_keys.expects(:resolve) + + apply + end + end + + context 'when #resolve raises an error' do + before do + Travis::Vault::Keys.stubs(:new).with(instance).returns(vault_keys) + vault_keys.stubs(:resolve).raises(Travis::Vault::RootKeyError) + instance.stubs(:sh).returns(sh) + end + + let(:sh) { stub('sh') } + + it do + sh.expects(:echo).with('Too many keys in fetched data. Probably you provided the root key. Terminating for security reasons.', ansi: :red) + sh.expects(:terminate) + + apply + end + end + end +end diff --git a/spec/build/git/clone_spec.rb b/spec/build/git/clone_spec.rb index 01c830240e..d816e01782 100644 --- a/spec/build/git/clone_spec.rb +++ b/spec/build/git/clone_spec.rb @@ -1,13 +1,13 @@ require 'spec_helper' -describe Travis::Build::Git::Clone, :sexp do +describe Travis::Vcs::Git::Clone, :sexp do let(:payload) { payload_for(:push, :ruby) } let(:script) { Travis::Build::Script.new(payload) } subject(:sexp) { script.sexp } let(:url) { "https://github.com/#{payload[:repository][:slug]}.git" } let(:dir) { payload[:repository][:slug] } - let(:depth) { Travis::Build::Git::DEFAULTS[:git][:depth] } + let(:depth) { Travis::Vcs::Git::DEFAULTS[:git][:depth] } let(:branch) { payload[:job][:branch] || 'master' } let(:oauth_token) { 'abcdef01234' } diff --git a/spec/build/git/ssh_key.rb b/spec/build/git/ssh_key.rb index 8bad2d47e2..12cf19a435 100644 --- a/spec/build/git/ssh_key.rb +++ b/spec/build/git/ssh_key.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Travis::Build::Git::SshKey, :sexp do +describe Travis::Vcs::Git::SshKey, :sexp do let(:payload) { payload_for(:push, :ruby) } let(:script) { Travis::Build::Script.new(payload) } subject { script.sexp } diff --git a/spec/build/git/submodules_spec.rb b/spec/build/git/submodules_spec.rb index 675cccacff..073c6c70bb 100644 --- a/spec/build/git/submodules_spec.rb +++ b/spec/build/git/submodules_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Travis::Build::Git::Clone, :sexp do +describe Travis::Vcs::Git::Clone, :sexp do let(:payload) { payload_for(:push, :ruby) } let(:script) { Travis::Build::Script.new(payload) } subject { script.sexp } diff --git a/spec/build/git/tarball_spec.rb b/spec/build/git/tarball_spec.rb index aca890c2de..7076687d9d 100644 --- a/spec/build/git/tarball_spec.rb +++ b/spec/build/git/tarball_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Travis::Build::Git::Clone, :sexp do +describe Travis::Vcs::Git::Clone, :sexp do let(:payload) { payload_for(:push, :ruby, oauth_token: 'secret') } let(:script) { Travis::Build::Script.new(payload) } subject { sexp_find(script.sexp, [:fold, 'git.tarball']) } diff --git a/spec/build/git_spec.rb b/spec/build/git_spec.rb index ede8d0b4ef..5e362dd8de 100644 --- a/spec/build/git_spec.rb +++ b/spec/build/git_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Travis::Build::Git, :sexp do +describe Travis::Vcs::Git, :sexp do let(:netrc_inst) { /echo -e "machine #{host}\\n login travis-ci\\n password access_token\\n" > \${TRAVIS_HOME}\/\.netrc/ } let(:netrc_oauth) { /echo -e "machine #{host}\\n login oauth_token\\n" > \${TRAVIS_HOME}\/\.netrc/ } let(:host) { 'github.com' } diff --git a/spec/build/services/vault/connect_spec.rb b/spec/build/services/vault/connect_spec.rb new file mode 100644 index 0000000000..ad50553bbd --- /dev/null +++ b/spec/build/services/vault/connect_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' + +describe Travis::Vault::Connect do + describe '#call' do + subject(:call) { described_class.call(vault) } + + let(:vault) do + { + api_url: 'https://myvault.org', + token: 'my-token' + } + end + + context 'the endpoint returns 200' do + before do + stub_request(:get, 'https://myvault.org/v1/auth/token/lookup-self'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 200) + end + + it { expect { call }.not_to raise_error } + end + + context 'the endpoint returns not-200' do + before do + stub_request(:get, 'https://myvault.org/v1/auth/token/lookup-self'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 403) + end + + it { expect { call }.to raise_error(Travis::Vault::ConnectionError) } + end + + context 'the endpoint is not correctly defined' do + let(:vault) do + { + api_url: '!https://myvault.org', + token: 'my-token' + } + end + + it { expect { call }.to raise_error(Travis::Vault::ConnectionError) } + end + end +end diff --git a/spec/build/services/vault/keys/build_paths_spec.rb b/spec/build/services/vault/keys/build_paths_spec.rb new file mode 100644 index 0000000000..8188922594 --- /dev/null +++ b/spec/build/services/vault/keys/build_paths_spec.rb @@ -0,0 +1,112 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::BuildPaths do + describe '#call' do + subject(:call) { described_class.new(secrets).call } + + context 'when everything is valid - some duplicates' do + let(:secrets) do + [ + { + namespace: [ + { name: 'ns1' }, + 'project_id/secret_key', + 'project_id/secret_key2' + ] + }, + { + namespace: [ + { name: 'ns2' }, + 'project_id2/secret_key', + 'project_id2/secret_key2' + ] + }, + 'ns1/project_id/secret_key', + 'ns1/project_id/secret_key2', + 'ns2/project_id/secret_key', + 'ns2/project_id/secret_key2', + ] + end + + it 'removes duplicates - it leaves only last secret_key and secret_key2 - + it is connected as env variables are defined - uses only last element of a path.' do + is_expected.to eq(%w[ns2/project_id2/secret_key ns2/project_id2/secret_key2 ns1/project_id/secret_key ns1/project_id/secret_key2 ns2/project_id/secret_key ns2/project_id/secret_key2]) + end + end + + context 'when everything is valid - no duplicates' do + let(:secrets) do + [ + { + namespace: [ + { name: 'ns1' }, + 'project_id/secret_key_a', + 'project_id/secret_key_b' + ] + }, + { + namespace: [ + { name: 'ns2' }, + 'project_id2/secret_key_c', + 'project_id2/secret_key2_d' + ] + }, + 'ns1/project_id/secret_key_e', + 'ns1/project_id/secret_key2_f', + 'ns2/project_id/secret_key_g', + 'ns2/project_id/secret_key2_h', + ] + end + + it do + is_expected.to eq(%w[ns1/project_id/secret_key_a ns1/project_id/secret_key_b ns2/project_id2/secret_key_c ns2/project_id2/secret_key2_d ns1/project_id/secret_key_e ns1/project_id/secret_key2_f ns2/project_id/secret_key_g ns2/project_id/secret_key2_h]) + end + end + + context 'when namespace key is not a namespace key' do + let(:secrets) do + [ + { + collection: [ + { name: 'ns1' }, + 'project_id/secret_key_a', + 'project_id/secret_key_b' + ] + }, + { + namespace: [ + { name: 'ns2' }, + 'project_id2/secret_key_c', + 'project_id2/secret_key2_d' + ] + } + ] + end + + it 'ignores unknown key' do + is_expected.to eq(%w[ns2/project_id2/secret_key_c ns2/project_id2/secret_key2_d]) + end + end + + context 'when namespace key is not a namespace key' do + let(:secrets) do + [ + { + namespace: %w[project_id/secret_key_a project_id/secret_key_b] + }, + { + namespace: [ + { name: 'ns2' }, + 'project_id2/secret_key_c', + 'project_id2/secret_key2_d' + ] + } + ] + end + + it 'is fine without namespace' do + is_expected.to eq(%w[project_id/secret_key_a project_id/secret_key_b ns2/project_id2/secret_key_c ns2/project_id2/secret_key2_d]) + end + end + end +end diff --git a/spec/build/services/vault/keys/kv1_spec.rb b/spec/build/services/vault/keys/kv1_spec.rb new file mode 100644 index 0000000000..bfd43b074e --- /dev/null +++ b/spec/build/services/vault/keys/kv1_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::KV1 do + describe '.resolve' do + subject { described_class.resolve(path, vault) } + + let(:vault) do + { + api_url: 'https://myvault.org', + token: 'my-token' + } + end + + let(:path) { 'path/to/variable' } + + context 'when the response code is 200' do + before do + stub_request(:get, 'https://myvault.org/v1/secret/path/to/variable'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 200, body: { data: { my_data: { b: '123' } } }.to_json) + end + + it do + is_expected.to eq({ 'my_data' => { 'b' => '123' } }) + end + end + + context 'when the response code is not 200' do + before do + stub_request(:get, 'https://myvault.org/v1/secret/path/to/variable'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 404, body: '') + end + + it 'does not explode' do + is_expected.to be_nil + end + end + end +end diff --git a/spec/build/services/vault/keys/kv2_spec.rb b/spec/build/services/vault/keys/kv2_spec.rb new file mode 100644 index 0000000000..57a8227263 --- /dev/null +++ b/spec/build/services/vault/keys/kv2_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::KV2 do + describe '.resolve' do + subject { described_class.resolve(path, vault) } + + let(:vault) do + { + api_url: 'https://myvault.org', + token: 'my-token' + } + end + + let(:path) { 'path/to/variable' } + + context 'when the response code is 200' do + before do + stub_request(:get, 'https://myvault.org/v1/secret/data/path/to/variable'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 200, body: { data: { data: { my_data: { a: '123' } } } }.to_json) + end + + it do + is_expected.to eq({ 'my_data' => { 'a' => '123' } }) + end + end + + context 'when the response code is not 200' do + before do + stub_request(:get, 'https://myvault.org/v1/secret/data/path/to/variable'). + with(headers: { 'X-Vault-Token': 'my-token' }). + to_return(status: 404, body: '') + end + + it 'does not explode' do + is_expected.to be_nil + end + end + end +end diff --git a/spec/build/services/vault/keys/paths_spec.rb b/spec/build/services/vault/keys/paths_spec.rb new file mode 100644 index 0000000000..fccf6a4794 --- /dev/null +++ b/spec/build/services/vault/keys/paths_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::Paths do + describe '.call' do + subject(:call) { described_class.call(vault) } + + let(:build_paths) { stub(call: nil) } + + let(:vault) do + { secrets: + [ + 'aaa/bbb', + 'ccc', + { kv_api_ver: 'kv1' }, + 'whatever/else/here' + ] + } + end + + it 'passes to BuildPaths initializer everything but not a hash with kv_api_key' do + Travis::Vault::Keys::BuildPaths.expects(:new).with(%w[aaa/bbb ccc whatever/else/here]).returns(build_paths) + build_paths.expects(:call) + + call + end + end +end diff --git a/spec/build/services/vault/keys/resolver_spec.rb b/spec/build/services/vault/keys/resolver_spec.rb new file mode 100644 index 0000000000..ad0489ad24 --- /dev/null +++ b/spec/build/services/vault/keys/resolver_spec.rb @@ -0,0 +1,63 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::Resolver do + describe '#call' do + let(:sh) { stub('sh') } + let(:vault) { stub('vault') } + let(:data) { stub('data') } + let(:appliance) { stub(sh: sh, vault: vault, data: data) } + let(:instance) { described_class.new(paths, 'kv2', appliance) } + + subject(:call) { instance.call } + + context 'when paths are empty' do + let(:paths) { [] } + + it 'does not call Vault' do + Travis::Vault::Keys::KV2.expects(:resolve).never + + call + end + end + + context 'when paths are not empty' do + let(:paths) { %w[path/to/something/secret_thing another/secret_thing another/secret_thing] } + + before do + Travis::Vault::Keys::KV2.stubs(:resolve).with(paths.first, vault).returns({ my_key: 'MySecretValue' }) + Travis::Vault::Keys::KV2.stubs(:resolve).with(paths[1], vault).returns({ something_else: 'ABC' }) + Travis::Vault::Keys::KV2.stubs(:resolve).with(paths.last, vault).returns({ something_else: 'ABC' }) + end + + context 'when path returns value from Vault' do + it do + sh.expects(:echo).never + sh.expects(:export).with('SECRET_THING_MY_KEY', %("MySecretValue"), echo: false, secure: true) + sh.expects(:export).with('SECRET_THING_SOMETHING_ELSE', %("ABC"), echo: false, secure: true).twice + data.expects(:vault_secrets=).with(%w[MySecretValue ABC]) + + call + end + end + + end + + context 'when path does not returns value from Vault' do + let(:paths) { %w[path/to/something/secret_thing another/secret_thing] } + + before do + Travis::Vault::Keys::KV2.stubs(:resolve).with(paths.first, vault).returns(nil) + Travis::Vault::Keys::KV2.stubs(:resolve).with(paths.last, vault).returns(nil) + end + + it do + sh.expects(:export).never + sh.expects(:echo).with('The value fetched for path/to/something/secret_thing is blank.', ansi: :yellow) + sh.expects(:echo).with('The value fetched for another/secret_thing is blank.', ansi: :yellow) + data.expects(:vault_secrets=).never + + call + end + end + end +end diff --git a/spec/build/services/vault/keys/version_spec.rb b/spec/build/services/vault/keys/version_spec.rb new file mode 100644 index 0000000000..01ec5e7152 --- /dev/null +++ b/spec/build/services/vault/keys/version_spec.rb @@ -0,0 +1,36 @@ +require 'spec_helper' + +describe Travis::Vault::Keys::Version do + describe '.call' do + subject(:call) { described_class.call(vault) } + + context 'when kv_api_ver key is there' do + let(:vault) do + { + secrets: [ + 'aaa/bbb', + 'ccc', + { kv_api_ver: 'kv1' }, + 'whatever/else/here' + ] + } + end + + it 'uses its value' do + is_expected.to eq('kv1') + end + end + + context 'when kv_api_ver is not there' do + let(:vault) do + { + secrets: %w[aaa/bbb ccc whatever/else/here] + } + end + + it 'uses default value' do + is_expected.to eq('kv2') + end + end + end +end diff --git a/spec/build/services/vault/keys_spec.rb b/spec/build/services/vault/keys_spec.rb new file mode 100644 index 0000000000..ca90976958 --- /dev/null +++ b/spec/build/services/vault/keys_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe Travis::Vault::Keys do + describe '#resolve' do + subject(:resolve) { described_class.new(appliance).resolve } + + let(:vault) { { api_url: 'https://my_vault.com', token: 'my_token' } } + let(:appliance) { stub(vault: vault) } + + let(:paths) { stub(:paths) } + let(:version) { stub(:version) } + let(:resolver) { stub(call: nil) } + + it 'calls Resolver with proper parameters' do + Travis::Vault::Keys::Version.expects(:call).with(vault).returns(version) + Travis::Vault::Keys::Paths.expects(:call).with(vault).returns(paths) + + Travis::Vault::Keys::Resolver.expects(:new).with(paths, version, appliance).returns(resolver) + resolver.expects(:call) + + resolve + end + end +end diff --git a/spec/build/vault_spec.rb b/spec/build/vault_spec.rb new file mode 100644 index 0000000000..40f5fd7916 --- /dev/null +++ b/spec/build/vault_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' + +describe "integration vault tests" do + before do + stub_request(:get, 'https://myvault.org/v1/secret/your/aaa/bbb'). + with(headers: { 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' }). + to_return(status: 200, body: { data: { my_data: { b: '123' } } }.to_json) + + stub_request(:get, 'https://myvault.org/v1/secret/my/x-something'). + with(headers: { 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' }). + to_return(status: 404, body: '') + + stub_request(:get, 'https://myvault.org/v1/secret/data/your/aaa/bbb'). + with(headers: { 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' }). + to_return(status: 200, body: { data: { my_data: { b: '123' } } }.to_json) + + stub_request(:get, 'https://myvault.org/v1/secret/data/my/x-something'). + with(headers: { 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' }). + to_return(status: 404, body: '') + end + + context 'when authenticated' do + before do + stub_request(:get, "https://myvault.org/v1/auth/token/lookup-self"). + with( + headers: { + 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' + }). + to_return(status: 200, body: "", headers: {}) + end + + %w[kv1 kv2].each do |version| + it do + expect do + Travis::Build::Script.new(JSON.parse(File.read("#{Dir.pwd}/spec/fixtures/build_config_with_vault_#{version}.json"))).sexp + end.not_to raise_error + end + end + end + + context 'when not authenticated' do + before do + stub_request(:get, "https://myvault.org/v1/auth/token/lookup-self"). + with( + headers: { + 'X-Vault-Token': 'hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD' + }). + to_return(status: 404, body: "", headers: {}) + end + + %w[kv1 kv2].each do |version| + it do + expect do + Travis::Build::Script.new(JSON.parse(File.read("#{Dir.pwd}/spec/fixtures/build_config_with_vault_#{version}.json"))).sexp + end.not_to raise_error + end + end + end +end diff --git a/spec/fixtures/build_config_with_vault_kv1.json b/spec/fixtures/build_config_with_vault_kv1.json new file mode 100644 index 0000000000..1576530238 --- /dev/null +++ b/spec/fixtures/build_config_with_vault_kv1.json @@ -0,0 +1,113 @@ +{ + "config": { + ".result": "configured", + "addons": { + "apt": { + "packages": [ + "bc" + ] + } + }, + "env": [ + "FOO=bar" + ], + "vault": { + "api_url": "https://myvault.org", + "token": "hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD", + "secrets": [ + { "kv_api_ver": "kv1" }, + { + "namespace": [ + {"name": "your"}, + "aaa/bbb" + ] + }, + "my/x-something" + ] + }, + "cache": "bundler", + "global_env": [ + "CHIRP_SUMMARY_OUTPUT=\"${TRAVIS_BUILD_DIR}/chirp.json\"", + "CHIRP_TRACKER_STATS_URL=\"https://chirp-tracker-staging.herokuapp.com/stats\"", + "SITE=org" + ], + "language": "ruby", + "linux_shared": { + "addons": { + "apt": { + "packages": [ + "ww" + ] + } + }, + "os": "linux" + }, + "notifications": { + "webhooks": { + "on_failure": "never", + "on_success": "always", + "urls": [ + "https://chirp-tracker-production.herokuapp.com/travis" + ] + } + }, + "os": "linux", + "rvm": "default", + "script": [ + "bundle exec rake" + ], + "sudo": false + }, + "env_vars": [ + { + "name": "hello", + "public": false, + "value": "yessy" + } + ], + "job": { + "allow_failure": false, + "branch": "master", + "commit": "a15c1259aeaf8e1955f01ed53abfa7cc5ef5e640", + "commit_message": "Bump dyno=scheduler.4396", + "commit_range": "f8e89280ead2...a15c1259aeaf", + "debug_options": {}, + "id": 539369, + "number": "18939.5", + "pull_request": false, + "queued_at": "2017-02-03T03:32:30Z", + "ref": null, + "secure_env_enabled": true, + "state": "queued", + "tag": null + }, + "paranoid": false, + "queue": "builds.macstadium6", + "repository": { + "api_url": "https://api.github.com/repos/travis-repos/chirp-org-staging", + "default_branch": "master", + "description": "chirp for staging.travis-ci.org", + "github_id": 30708062, + "id": 17192, + "last_build_duration": 15, + "last_build_finished_at": "2017-02-03T03:28:27Z", + "last_build_id": 539357, + "last_build_number": "18938", + "last_build_started_at": "2017-02-03T03:23:02Z", + "last_build_state": "canceled", + "slug": "travis-repos/chirp-org-staging", + "source_url": "https://github.com/travis-repos/chirp-org-staging.git" + }, + "source": { + "event_type": "push", + "id": 539364, + "number": "18939" + }, + "ssh_key": null, + "timeouts": { + "hard_limit": null, + "log_silence": null + }, + "type": "test", + "vm_type": "default" +} diff --git a/spec/fixtures/build_config_with_vault_kv2.json b/spec/fixtures/build_config_with_vault_kv2.json new file mode 100644 index 0000000000..07495a3172 --- /dev/null +++ b/spec/fixtures/build_config_with_vault_kv2.json @@ -0,0 +1,114 @@ +{ + "config": { + ".result": "configured", + "addons": { + "apt": { + "packages": [ + "bc" + ] + } + }, + "env": [ + "FOO=bar" + ], + "vault": { + "api_url": "https://myvault.org", + "token": "hvs.Pgfcl9Nr0AozXCLQF5Wtb6FSD", + "secrets": [ + { + "namespace": [ + { + "name": "your" + }, + "aaa/bbb" + ] + }, + "my/x-something" + ] + }, + "cache": "bundler", + "global_env": [ + "CHIRP_SUMMARY_OUTPUT=\"${TRAVIS_BUILD_DIR}/chirp.json\"", + "CHIRP_TRACKER_STATS_URL=\"https://chirp-tracker-staging.herokuapp.com/stats\"", + "SITE=org" + ], + "language": "ruby", + "linux_shared": { + "addons": { + "apt": { + "packages": [ + "ww" + ] + } + }, + "os": "linux" + }, + "notifications": { + "webhooks": { + "on_failure": "never", + "on_success": "always", + "urls": [ + "https://chirp-tracker-production.herokuapp.com/travis" + ] + } + }, + "os": "linux", + "rvm": "default", + "script": [ + "bundle exec rake" + ], + "sudo": false + }, + "env_vars": [ + { + "name": "hello", + "public": false, + "value": "yessy" + } + ], + "job": { + "allow_failure": false, + "branch": "master", + "commit": "a15c1259aeaf8e1955f01ed53abfa7cc5ef5e640", + "commit_message": "Bump dyno=scheduler.4396", + "commit_range": "f8e89280ead2...a15c1259aeaf", + "debug_options": {}, + "id": 539369, + "number": "18939.5", + "pull_request": false, + "queued_at": "2017-02-03T03:32:30Z", + "ref": null, + "secure_env_enabled": true, + "state": "queued", + "tag": null + }, + "paranoid": false, + "queue": "builds.macstadium6", + "repository": { + "api_url": "https://api.github.com/repos/travis-repos/chirp-org-staging", + "default_branch": "master", + "description": "chirp for staging.travis-ci.org", + "github_id": 30708062, + "id": 17192, + "last_build_duration": 15, + "last_build_finished_at": "2017-02-03T03:28:27Z", + "last_build_id": 539357, + "last_build_number": "18938", + "last_build_started_at": "2017-02-03T03:23:02Z", + "last_build_state": "canceled", + "slug": "travis-repos/chirp-org-staging", + "source_url": "https://github.com/travis-repos/chirp-org-staging.git" + }, + "source": { + "event_type": "push", + "id": 539364, + "number": "18939" + }, + "ssh_key": null, + "timeouts": { + "hard_limit": null, + "log_silence": null + }, + "type": "test", + "vm_type": "default" +} diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 01a31c1584..1e6799992a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,6 +5,9 @@ require 'sinatra/test_helpers' require 'travis/build' require 'pathname' +require 'webmock/rspec' + +WebMock.allow_net_connect! Dir["{spec/spec_helpers,spec/support,spec/**/shared}/**/*.rb"].each do |f| load(f) diff --git a/spec/support/payloads.rb b/spec/support/payloads.rb index 691ce399ed..30fe947945 100644 --- a/spec/support/payloads.rb +++ b/spec/support/payloads.rb @@ -9,7 +9,8 @@ 'config' => { 'os' => 'linux', 'arch' => 'amd64', - 'env' => ['FOO=foo', 'SECURE BAR=bar'] + 'env' => ['FOO=foo', 'SECURE BAR=bar'], + 'server_type' => 'git' }, 'repository' => { 'github_id' => 42, @@ -39,7 +40,8 @@ 'host' => 'travis-ci.com', 'config' => { 'os' => 'linux', - 'env' => ['FOO=foo', 'SECURE BAR=bar'] + 'env' => ['FOO=foo', 'SECURE BAR=bar'], + 'server_type' => 'git' }, 'repository' => { 'github_id' => 42, From bd5c57630ba0526b722ed7fc5e2ea822c9a78b2b Mon Sep 17 00:00:00 2001 From: gabriel-arc <57348209+GbArc@users.noreply.github.com> Date: Wed, 28 Dec 2022 10:21:36 +0100 Subject: [PATCH 24/31] sinatra 2.2.3 (#2057) --- Gemfile.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 683e29b4ec..3729342936 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -109,7 +109,7 @@ GEM msgpack (1.2.4) multi_json (1.15.0) multipart-post (2.2.3) - mustermann (1.1.1) + mustermann (2.0.2) ruby2_keywords (~> 0.0.1) net-http-persistent (3.0.0) connection_pool (~> 2.2) @@ -135,8 +135,8 @@ GEM pusher-client (0.6.2) json websocket (~> 1.0) - rack (2.2.3.1) - rack-protection (2.2.0) + rack (2.2.4) + rack-protection (2.2.3) rack rack-ssl (1.4.1) rack @@ -194,19 +194,19 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sinatra (2.2.0) - mustermann (~> 1.0) + sinatra (2.2.3) + mustermann (~> 2.0) rack (~> 2.2) - rack-protection (= 2.2.0) + rack-protection (= 2.2.3) tilt (~> 2.0) - sinatra-contrib (2.2.0) + sinatra-contrib (2.2.3) multi_json - mustermann (~> 1.0) - rack-protection (= 2.2.0) - sinatra (= 2.2.0) + mustermann (~> 2.0) + rack-protection (= 2.2.3) + sinatra (= 2.2.3) tilt (~> 2.0) thread_safe (0.3.6) - tilt (2.0.10) + tilt (2.0.11) travis (1.8.9) backports faraday (~> 0.9) From 3d51b9c71355dce711aa2d6e001012fcbc28e253 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Wed, 28 Dec 2022 10:58:49 +0100 Subject: [PATCH 25/31] ghc update [ship:docker] --- public/version-aliases/ghc.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index da5d0b2239..d286dcaecb 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,9 +166,9 @@ "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.4.3", - "9.x": "9.4.3", - "9.x.x": "9.4.3", + "9": "9.4.4", + "9.x": "9.4.4", + "9.x.x": "9.4.4", "9.0.x": "9.0.2", "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", @@ -181,11 +181,12 @@ "9.2.4": "9.2.4", "9.2.5": "9.2.5", "9.4.1-alpha1": "9.4.1-alpha1", - "9.4": "9.4.3", + "9.4": "9.4.4", "9.4.1-alpha2": "9.4.1-alpha2", "9.4.1-alpha3": "9.4.1-alpha3", "9.4.1": "9.4.1", - "9.4.x": "9.4.3", + "9.4.x": "9.4.4", "9.4.2": "9.4.2", - "9.4.3": "9.4.3" + "9.4.3": "9.4.3", + "9.4.4": "9.4.4" } \ No newline at end of file From 8842a837aadde4bf9a7997fa5a51ce36ea449fa4 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Tue, 10 Jan 2023 10:43:25 +0100 Subject: [PATCH 26/31] added cri-dockerd service --- lib/travis/build/appliances/services.rb | 15 ++++++++++- .../build/bash/travis_setup_cri-dockerd.bash | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 lib/travis/build/bash/travis_setup_cri-dockerd.bash diff --git a/lib/travis/build/appliances/services.rb b/lib/travis/build/appliances/services.rb index 9e25cd29ae..2148be495f 100644 --- a/lib/travis/build/appliances/services.rb +++ b/lib/travis/build/appliances/services.rb @@ -10,7 +10,8 @@ class Services < Base 'memcache' => 'memcached', 'neo4j-server' => 'neo4j', 'rabbitmq' => 'rabbitmq-server', - 'redis' => 'redis-server' + 'redis' => 'redis-server', + 'cri-dockerd' => 'cri_dockerd' } def apply @@ -20,7 +21,9 @@ def apply sh.else do sh.fold 'services' do services.each do |name| + puts "SVC: #{name.inspect}" service_apply_method = "apply_#{name}" + puts "meth: #{service_apply_method}" if respond_to?(service_apply_method) send(service_apply_method) next @@ -56,6 +59,16 @@ def apply_mongodb end end + def apply_cri_dockerd + sh.if '"$TRAVIS_OS_NAME" != linux' do + sh.echo "Addon cri-dockerd is not supported on #{data[:config][:os]}", ansi: :red + end + sh.else do + sh.raw bash('travis_setup_cri-dockerd'), echo: false, timing: false + sh.cmd "travis_setup_cri-dockerd", echo: true, timing: true + end + end + def apply_mysql sh.raw <<~BASH travis_mysql_ping() { diff --git a/lib/travis/build/bash/travis_setup_cri-dockerd.bash b/lib/travis/build/bash/travis_setup_cri-dockerd.bash new file mode 100644 index 0000000000..3514acfb28 --- /dev/null +++ b/lib/travis/build/bash/travis_setup_cri-dockerd.bash @@ -0,0 +1,26 @@ +travis_setup_cri-dockerd() { + local cri_containerd_cni_version='1.6.14' + local crictl_version='v1.24.2' + echo -e "${ANSI_YELLOW}cri-dockerd setup ${ANSI_CLEAR}" + sudo bash -c " + groupadd docker || true; + apt-get update && apt-get install socat eptables; + apt-get install conntrack containerd; + wget https://github.com/containerd/containerd/releases/download/v${cri_containerd_cni_version}/cri-containerd-cni-${cri_containerd_cni_version}-linux-amd64.tar.gz; + tar zxvf cri-containerd-cni-${cri_containerd_cni_version}-linux-amd64.tar.gz -C /; + rm -rf cri-containerd-cni-1.6.14-linux-amd64.tar.gz; + wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.0/cri-dockerd_0.3.0.3-0.ubuntu-bionic_amd64.deb; + dpkg -i cri-dockerd_0.3.0.3-0.ubuntu-bionic_amd64.deb; + wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/v0.3.0/packaging/systemd/cri-docker.service + wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/v0.3.0/packaging/systemd/cri-docker.socket; + mv cri-docker.socket cri-docker.service /etc/systemd/system/; + systemctl daemon-reload; + systemctl enable cri-docker.service; + systemctl enable --now cri-docker.socket; + wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${crictl_version}/crictl-${crictl_version}-linux-amd64.tar.gz + tar zxvf crictl-${crictl_version}-linux-amd64.tar.gz -C /usr/bin; + rm -f crictl-${crictl_version}-linux-amd64.tar.gz; + echo runtime-endpoint: unix:///run/containerd/containerd.sock > /etc/crictl.yaml; + echo image-endpoint: unix:///run/containerd/containerd.sock >> /etc/crictl.yaml; +"; +} From 89bdbbef0852b6091980e10add033230cbd1df0f Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Tue, 10 Jan 2023 10:52:08 +0100 Subject: [PATCH 27/31] shfmt fix, dbg removed --- lib/travis/build/appliances/services.rb | 2 -- lib/travis/build/bash/travis_setup_cri-dockerd.bash | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/travis/build/appliances/services.rb b/lib/travis/build/appliances/services.rb index 2148be495f..1a96718684 100644 --- a/lib/travis/build/appliances/services.rb +++ b/lib/travis/build/appliances/services.rb @@ -21,9 +21,7 @@ def apply sh.else do sh.fold 'services' do services.each do |name| - puts "SVC: #{name.inspect}" service_apply_method = "apply_#{name}" - puts "meth: #{service_apply_method}" if respond_to?(service_apply_method) send(service_apply_method) next diff --git a/lib/travis/build/bash/travis_setup_cri-dockerd.bash b/lib/travis/build/bash/travis_setup_cri-dockerd.bash index 3514acfb28..72b9e0a4de 100644 --- a/lib/travis/build/bash/travis_setup_cri-dockerd.bash +++ b/lib/travis/build/bash/travis_setup_cri-dockerd.bash @@ -22,5 +22,5 @@ travis_setup_cri-dockerd() { rm -f crictl-${crictl_version}-linux-amd64.tar.gz; echo runtime-endpoint: unix:///run/containerd/containerd.sock > /etc/crictl.yaml; echo image-endpoint: unix:///run/containerd/containerd.sock >> /etc/crictl.yaml; -"; +" } From a88aa71351f5c919e8bd6660976c82120ab6ac79 Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Fri, 27 Jan 2023 15:04:10 +0100 Subject: [PATCH 28/31] removed npm spinner config --- lib/travis/build/script/node_js.rb | 1 - spec/build/script/node_js_spec.rb | 4 ---- 2 files changed, 5 deletions(-) diff --git a/lib/travis/build/script/node_js.rb b/lib/travis/build/script/node_js.rb index acacf6ffd7..be53c65b6a 100644 --- a/lib/travis/build/script/node_js.rb +++ b/lib/travis/build/script/node_js.rb @@ -31,7 +31,6 @@ def setup sh.newline npm_disable_prefix - npm_disable_spinner npm_disable_progress npm_disable_strict_ssl unless npm_strict_ssl? install_yarn_when_locked diff --git a/spec/build/script/node_js_spec.rb b/spec/build/script/node_js_spec.rb index 12a6d0418e..d16858b610 100644 --- a/spec/build/script/node_js_spec.rb +++ b/spec/build/script/node_js_spec.rb @@ -111,10 +111,6 @@ should include_sexp [:cmd, 'npm --version', echo: true] end - it 'disables the npm spinner' do - should include_sexp [:cmd, 'npm config set spin false', assert: true] - end - describe 'if package.json exists' do let(:sexp) { sexp_find(subject, [:if, '-f package.json'], [:then]) } From 6b39ff8a8d3f190f68bf084cc0d7d79315125bdf Mon Sep 17 00:00:00 2001 From: gabriel-arc Date: Mon, 30 Jan 2023 10:37:16 +0100 Subject: [PATCH 29/31] ghc up --- public/version-aliases/ghc.json | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/public/version-aliases/ghc.json b/public/version-aliases/ghc.json index d286dcaecb..d35ed379a3 100644 --- a/public/version-aliases/ghc.json +++ b/public/version-aliases/ghc.json @@ -166,9 +166,9 @@ "9.0.1-alpha1": "9.0.1-alpha1", "9.0": "9.0.2", "9.0.1": "9.0.1", - "9": "9.4.4", - "9.x": "9.4.4", - "9.x.x": "9.4.4", + "9": "9.6.0.20230128", + "9.x": "9.6.0.20230128", + "9.x.x": "9.6.0.20230128", "9.0.x": "9.0.2", "9.0.2": "9.0.2", "9.2.1-alpha1": "9.2.1-alpha1", @@ -188,5 +188,11 @@ "9.4.x": "9.4.4", "9.4.2": "9.4.2", "9.4.3": "9.4.3", - "9.4.4": "9.4.4" + "9.4.4": "9.4.4", + "9.6.0.20230111": "9.6.0.20230111", + "9.6.x": "9.6.0.20230128", + "9.6": "9.6.1-alpha2", + "9.6.0.20230128": "9.6.0.20230128", + "9.6.1-alpha1": "9.6.1-alpha1", + "9.6.1-alpha2": "9.6.1-alpha2" } \ No newline at end of file From ca62053e9f55071f5589f977f0ada3b6d135e97b Mon Sep 17 00:00:00 2001 From: GbArc Date: Wed, 31 May 2023 12:46:34 +0200 Subject: [PATCH 30/31] mtu changed to 1350 --- .../build/appliances/set_docker_mtu_and_registry_mirrors.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb b/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb index 58b7aa77c6..23d945df15 100644 --- a/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb +++ b/lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb @@ -16,11 +16,11 @@ def apply sh.raw <<-EOF sudo test -f /etc/docker/daemon.json if [[ $? = 0 ]]; then - echo '[{"op":"add","path":"/mtu","value":1460}]' > mtu.jsonpatch + echo '[{"op":"add","path":"/mtu","value":1350}]' > mtu.jsonpatch sudo jsonpatch /etc/docker/daemon.json mtu.jsonpatch > daemon.json sudo mv daemon.json /etc/docker/daemon.json else - echo '{"mtu":1460}' | sudo tee /etc/docker/daemon.json > /dev/null + echo '{"mtu":1350}' | sudo tee /etc/docker/daemon.json > /dev/null fi if curl --connect-timeout 1 -fsSL -o /dev/null \ From 6dbe88f67d9e06f63c05e21c94e75db8b8e48989 Mon Sep 17 00:00:00 2001 From: Muqtader-MKD <123384571+mkdtravis@users.noreply.github.com> Date: Mon, 10 Jul 2023 21:05:45 +0500 Subject: [PATCH 31/31] fix for SUPP-712 --- lib/travis/build/appliances/services.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/build/appliances/services.rb b/lib/travis/build/appliances/services.rb index 1a96718684..6af2b8e117 100644 --- a/lib/travis/build/appliances/services.rb +++ b/lib/travis/build/appliances/services.rb @@ -47,7 +47,7 @@ def apply_mongodb sh.cmd 'sudo service mongodb start', echo: true, timing: true end sh.elif '"$TRAVIS_DIST" == focal' do - sh.cmd 'sudo systemctl start mongodb', echo: true, timing: true + sh.cmd 'sudo systemctl start mongod', echo: true, timing: true end sh.elif '"$TRAVIS_INIT" == upstart' do sh.cmd 'sudo service mongod start', echo: true, timing: true