From 48c4142a6b18fdf29046f3702738bbfeec5c47ac Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Wed, 8 May 2024 09:51:40 +0100 Subject: [PATCH 01/50] add the service in redborder full installation --- resources/attributes/default.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 6477e968..ff290795 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -106,12 +106,12 @@ default["redborder"]["memory_assigned"] = {} # geoip has been removed because is not a service -default["redborder"]["services_group"]["full"] = %w[consul chef-server zookeeper memcached rsyslog kafka logstash s3 - postgresql redborder-postgresql nginx webui druid-broker - druid-historical druid-realtime druid-coordinator f2k - redborder-monitor pmacct redborder-dswatcher - redborder-events-counter http2k - ] +default["redborder"]["services_group"]["full"] = %w[ + consul chef-server zookeeper memcached rsyslog kafka logstash s3 + redborder-ale postgresql redborder-postgresql nginx webui druid-broker + druid-historical druid-realtime druid-coordinator f2k redborder-monitor + pmacct redborder-dswatcher redborder-events-counter http2k + ] default["redborder"]["services_group"]["custom"] = [] default["redborder"]["services_group"]["core"] = %w[consul zookeeper druid-coordinator druid-overlord hadoop-resourcemanager] #consul server @@ -167,7 +167,7 @@ default["redborder"]["services"]["redborder-events-counter"] = false default["redborder"]["services"]["rsyslog"] = true default["redborder"]["services"]["redborder-nmsp"] = false -default["redborder"]["services"]["redborder-ale"] = false +default["redborder"]["services"]["redborder-ale"] = true default["redborder"]["services"]["n2klocd"] = false default["redborder"]["services"]["radiusd"] = false From 0a4aaa731b094f5c81a2ec35723bcccba8f9158e Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Thu, 9 May 2024 13:35:21 +0100 Subject: [PATCH 02/50] adding sudo permissions thorugh this cookbook to ale service --- resources/templates/default/redborder-manager.erb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/resources/templates/default/redborder-manager.erb b/resources/templates/default/redborder-manager.erb index 35cd16b7..3f420f56 100644 --- a/resources/templates/default/redborder-manager.erb +++ b/resources/templates/default/redborder-manager.erb @@ -6,3 +6,9 @@ Defaults:hadoop !requiretty redborder ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_wakeup_chef.sh, /opt/rb/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh redborder-monitor ALL= NOPASSWD: /usr/lib/redborder/bin/rb_get_sensor.sh, /usr/lib/redborder/bin/cachestat.sh, /bin/nice -n 19 /usr/sbin/fping -p 1 -c 10 *, /usr/lib/redborder/bin/rb_mem.sh * webui ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_get_raw_vault.sh *, /usr/lib/redborder/bin/rb_update_geodb_cookbook.sh, /usr/lib/redborder/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef, /usr/lib/redborder/bin/rb_reassign_partitions.sh -d -e, /usr/lib/redborder/bin/rb_pcp.sh up *, /usr/lib/redborder/bin/rb_pcp.sh recovery *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_pcp.sh status, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_get_zkinfo.sh, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_create_bulkstats_columns.sh *, /usr/lib/redborder/bin/rb_create_rsa.sh * +<%=" +redborder-ale + ALL=NOPASSWD:SETENV: + /usr/lib/rvm/rubies/ruby-2.7.5/bin/ruby, + /var/rb-ale/bin/rb_ale.rb +".gsub(/\s+/, ' ').strip %> \ No newline at end of file From 70dd4ac084fec2cb5955e5ce8f683be0af952460 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Thu, 16 May 2024 14:15:43 +0100 Subject: [PATCH 03/50] Bump to 1.7.10 --- CHANGELOG.md | 5 +++++ resources/metadata.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adb6ada5..01cd72f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ cookbook-rb-manager CHANGELOG =============== +## 1.7.10 + + - nilsver + - [f66e148] added conditional check on pipelines + ## 1.7.9 - Miguel Álvarez diff --git a/resources/metadata.rb b/resources/metadata.rb index 9a875064..2b7fab4a 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.7.9' +version '1.7.10' depends 'chef-server' depends 'zookeeper' From d3defac209d3b81c95b7838b09967dd143bdf202 Mon Sep 17 00:00:00 2001 From: nilsver Date: Fri, 17 May 2024 08:25:55 +0100 Subject: [PATCH 04/50] release 1.7.11 --- CHANGELOG.md | 11 +++++++++++ resources/metadata.rb | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01cd72f6..a844b3fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ cookbook-rb-manager CHANGELOG =============== +## 1.7.11 + + - Luis Blanco + - [6692d79] Merge branch 'development' into feature/#16519_integrate_rb_ale + - [f8570d2] Merge branch 'development' into feature/#16519_integrate_rb_ale + - [20c6eec] Merge remote-tracking branch 'origin/development' into feature/#16519_integrate_rb_ale + - [0a4aaa7] adding sudo permissions thorugh this cookbook to ale service + - [4542a60] Merge branch 'development' into feature/#16519_integrate_rb_ale + - [233c7c2] Merge remote-tracking branch 'origin/master' into development + - [48c4142] add the service in redborder full installation + ## 1.7.10 - nilsver diff --git a/resources/metadata.rb b/resources/metadata.rb index 2b7fab4a..81ff87a3 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.7.10' +version '1.7.11' depends 'chef-server' depends 'zookeeper' From c6a660ac39024fc942d8a9fe6667e136b823d877 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Fri, 17 May 2024 10:03:10 +0100 Subject: [PATCH 05/50] Revert "adding sudo permissions thorugh this cookbook to ale service" This reverts commit 0a4aaa731b094f5c81a2ec35723bcccba8f9158e. --- resources/templates/default/redborder-manager.erb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/resources/templates/default/redborder-manager.erb b/resources/templates/default/redborder-manager.erb index 3f420f56..35cd16b7 100644 --- a/resources/templates/default/redborder-manager.erb +++ b/resources/templates/default/redborder-manager.erb @@ -6,9 +6,3 @@ Defaults:hadoop !requiretty redborder ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_wakeup_chef.sh, /opt/rb/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh redborder-monitor ALL= NOPASSWD: /usr/lib/redborder/bin/rb_get_sensor.sh, /usr/lib/redborder/bin/cachestat.sh, /bin/nice -n 19 /usr/sbin/fping -p 1 -c 10 *, /usr/lib/redborder/bin/rb_mem.sh * webui ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_get_raw_vault.sh *, /usr/lib/redborder/bin/rb_update_geodb_cookbook.sh, /usr/lib/redborder/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef, /usr/lib/redborder/bin/rb_reassign_partitions.sh -d -e, /usr/lib/redborder/bin/rb_pcp.sh up *, /usr/lib/redborder/bin/rb_pcp.sh recovery *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_pcp.sh status, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_get_zkinfo.sh, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_create_bulkstats_columns.sh *, /usr/lib/redborder/bin/rb_create_rsa.sh * -<%=" -redborder-ale - ALL=NOPASSWD:SETENV: - /usr/lib/rvm/rubies/ruby-2.7.5/bin/ruby, - /var/rb-ale/bin/rb_ale.rb -".gsub(/\s+/, ' ').strip %> \ No newline at end of file From 1d92d54363133fa1e4e42b2a95faedf8b24087cd Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Fri, 17 May 2024 10:06:22 +0100 Subject: [PATCH 06/50] revert redborder-ale by default in attributes --- resources/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 08d33f3e..fc2d8deb 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -170,7 +170,7 @@ default["redborder"]["services"]["redborder-events-counter"] = false default["redborder"]["services"]["rsyslog"] = true default["redborder"]["services"]["redborder-nmsp"] = false -default["redborder"]["services"]["redborder-ale"] = true +default["redborder"]["services"]["redborder-ale"] = false default["redborder"]["services"]["n2klocd"] = false default["redborder"]["services"]["radiusd"] = false From c2df76acc335f36b1abcd8d5fa3d22bc22afaf71 Mon Sep 17 00:00:00 2001 From: David Vanhoucke Date: Fri, 17 May 2024 09:34:02 +0000 Subject: [PATCH 07/50] add postfix service --- resources/attributes/default.rb | 2 ++ resources/metadata.rb | 1 + resources/recipes/configure.rb | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 27135854..2a54bfaf 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -173,6 +173,7 @@ default["redborder"]["services"]["redborder-ale"] = false default["redborder"]["services"]["n2klocd"] = false default["redborder"]["services"]["radiusd"] = false +default["redborder"]["services"]["postfix"] = true default["redborder"]["systemdservices"]["chef-client"] = ["chef-client"] default["redborder"]["systemdservices"]["chef-server"] = ["opscode-erchef"] @@ -209,6 +210,7 @@ default["redborder"]["systemdservices"]["redborder-ale"] = ["redborder-ale"] default["redborder"]["systemdservices"]["n2klocd"] = ["n2klocd"] default["redborder"]["systemdservices"]["radiusd"] = ["radiusd"] +default["redborder"]["systemdservices"]["postfix"] = ["postfix"] # Tier default["redborder"]["druid"]["historical"]["tier"] = "default" diff --git a/resources/metadata.rb b/resources/metadata.rb index 2b7fab4a..d9559b9f 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -42,3 +42,4 @@ depends 'rbcgroup' depends 'rblogstatter' depends 'rb-arubacentral' +depends 'rb-postfix' \ No newline at end of file diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 2585a38e..77e453ff 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -355,6 +355,10 @@ action (node["redborder"]["services"]["redborder-cep"] ? [:add, :register] : [:remove, :deregister]) end +rb_postfix_config "Configure postfix" do + action (node["redborder"]["services"]["postfix"] ? :add : :remove) +end + rbcgroup_config "Configure cgroups" do action :add end From b1a390380eb06e40ea1bd65553afa6868d3a503b Mon Sep 17 00:00:00 2001 From: nilsver Date: Fri, 17 May 2024 14:13:40 +0100 Subject: [PATCH 08/50] update changelog --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a844b3fc..1740f907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,6 @@ cookbook-rb-manager CHANGELOG ## 1.7.11 - Luis Blanco - - [6692d79] Merge branch 'development' into feature/#16519_integrate_rb_ale - - [f8570d2] Merge branch 'development' into feature/#16519_integrate_rb_ale - - [20c6eec] Merge remote-tracking branch 'origin/development' into feature/#16519_integrate_rb_ale - - [0a4aaa7] adding sudo permissions thorugh this cookbook to ale service - - [4542a60] Merge branch 'development' into feature/#16519_integrate_rb_ale - - [233c7c2] Merge remote-tracking branch 'origin/master' into development - [48c4142] add the service in redborder full installation ## 1.7.10 From 5cdb6d9baa193f0697d3aca426de4e21f48ff2ae Mon Sep 17 00:00:00 2001 From: nilsver Date: Fri, 17 May 2024 14:19:24 +0100 Subject: [PATCH 09/50] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1740f907..04806bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,7 @@ cookbook-rb-manager CHANGELOG =============== -## 1.7.11 - +## 1.8.0 - Luis Blanco - [48c4142] add the service in redborder full installation From dd37cbbdb9982ca9c9b01d11428c24758a71a966 Mon Sep 17 00:00:00 2001 From: nilsver Date: Fri, 17 May 2024 14:21:26 +0100 Subject: [PATCH 10/50] update metadata.rb --- resources/metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/metadata.rb b/resources/metadata.rb index 81ff87a3..9e5df275 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.7.11' +version '1.8.0' depends 'chef-server' depends 'zookeeper' From 0674093f771aac8b13f6ad7f8acedf6872a0fdf5 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Mon, 20 May 2024 17:26:19 +0100 Subject: [PATCH 11/50] version bumped, code linted --- resources/metadata.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/metadata.rb b/resources/metadata.rb index e1c9c9b1..b8f3a44e 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.8.0' +version '1.9.0' depends 'chef-server' depends 'zookeeper' @@ -36,10 +36,10 @@ depends 'freeradius' depends 'rbcep' depends 'cron' -#depends 'ohai' +# depends 'ohai' depends 'rb-selinux' depends 'rbaioutliers' depends 'rbcgroup' depends 'rblogstatter' depends 'rb-arubacentral' -depends 'rb-postfix' \ No newline at end of file +depends 'rb-postfix' From b60f9ae033b3b4ff3463e7340da47a9010eca379 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Mon, 20 May 2024 17:26:48 +0100 Subject: [PATCH 12/50] changelog clean and described new changes --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04806bc3..aff31a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ cookbook-rb-manager CHANGELOG =============== +## 1.9.0 + - David Vanhoucke + - [c2df76a] add postfix service + ## 1.8.0 - Luis Blanco - - [48c4142] add the service in redborder full installation + - [48c4142] add ale service in redborder full installation ## 1.7.10 From b487006aebc2cb69e8f5bacb4094ceb70e0e3dc3 Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Tue, 21 May 2024 10:44:16 +0100 Subject: [PATCH 13/50] Update CHANGELOG.md --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aff31a0d..b77bb792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,8 @@ cookbook-rb-manager CHANGELOG =============== - -## 1.9.0 +## 1.8.0 - David Vanhoucke - [c2df76a] add postfix service - -## 1.8.0 - Luis Blanco - [48c4142] add ale service in redborder full installation From 157ae8d1e330770bb45bbc0744f8dcec461eb80b Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Tue, 21 May 2024 10:44:39 +0100 Subject: [PATCH 14/50] Update metadata.rb --- resources/metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/metadata.rb b/resources/metadata.rb index b8f3a44e..f5956ab7 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.9.0' +version '1.8.0' depends 'chef-server' depends 'zookeeper' From 4484faa0c170ea5260092fb6012d137cdfe94f5c Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Tue, 21 May 2024 10:51:35 +0100 Subject: [PATCH 15/50] Update default.rb Agree that redborder-ale is not a service to be in the full installation of the manager --- resources/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 5bbfffc0..6be2469e 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -111,7 +111,7 @@ # geoip has been removed because is not a service default["redborder"]["services_group"]["full"] = %w[ consul chef-server zookeeper memcached rsyslog kafka logstash s3 - redborder-ale postgresql redborder-postgresql nginx webui druid-broker + postgresql redborder-postgresql nginx webui druid-broker druid-historical druid-realtime druid-coordinator f2k redborder-monitor pmacct redborder-dswatcher redborder-events-counter http2k ] From 61671ea5b0e330a0f6d17c4d4cc5f08023b205eb Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Tue, 21 May 2024 10:58:27 +0100 Subject: [PATCH 16/50] Update default.rb Unlint due to normalization --- resources/attributes/default.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 6be2469e..c8883595 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -110,11 +110,11 @@ # geoip has been removed because is not a service default["redborder"]["services_group"]["full"] = %w[ - consul chef-server zookeeper memcached rsyslog kafka logstash s3 - postgresql redborder-postgresql nginx webui druid-broker - druid-historical druid-realtime druid-coordinator f2k redborder-monitor - pmacct redborder-dswatcher redborder-events-counter http2k - ] + consul chef-server zookeeper memcached rsyslog kafka logstash s3 + postgresql redborder-postgresql nginx webui druid-broker + druid-historical druid-realtime druid-coordinator f2k redborder-monitor + pmacct redborder-dswatcher redborder-events-counter http2k + ] default["redborder"]["services_group"]["custom"] = [] default["redborder"]["services_group"]["core"] = %w[consul zookeeper druid-coordinator druid-overlord hadoop-resourcemanager] #consul server From 8b6e02e682f5c37b0ba2548e8e684f1ede32c434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Tue, 21 May 2024 11:08:33 +0100 Subject: [PATCH 17/50] Update default.rb --- resources/attributes/default.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index c8883595..2a54bfaf 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -109,11 +109,11 @@ default["redborder"]["memory_assigned"] = {} # geoip has been removed because is not a service -default["redborder"]["services_group"]["full"] = %w[ - consul chef-server zookeeper memcached rsyslog kafka logstash s3 +default["redborder"]["services_group"]["full"] = %w[consul chef-server zookeeper memcached rsyslog kafka logstash s3 postgresql redborder-postgresql nginx webui druid-broker - druid-historical druid-realtime druid-coordinator f2k redborder-monitor - pmacct redborder-dswatcher redborder-events-counter http2k + druid-historical druid-realtime druid-coordinator f2k + redborder-monitor pmacct redborder-dswatcher + redborder-events-counter http2k ] default["redborder"]["services_group"]["custom"] = [] From ef94e3cfee08228f11473f9ac28632f3a8d8e636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Negr=C3=B3n?= <45871721+manegron@users.noreply.github.com> Date: Fri, 24 May 2024 12:02:24 +0100 Subject: [PATCH 18/50] Add configure common cookbook call (#140) --- resources/metadata.rb | 1 + resources/recipes/configure.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/resources/metadata.rb b/resources/metadata.rb index f5956ab7..2ee9625a 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -6,6 +6,7 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) version '1.8.0' +depends 'rb-common' depends 'chef-server' depends 'zookeeper' depends 'kafka' diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 77e453ff..617840fc 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -12,6 +12,10 @@ # manager services manager_services = manager_services() +rb_common_config "Configure common" do + action :configure +end + rb_selinux_config "Configure Selinux" do if shell_out("getenforce").stdout.chomp == "Disabled" action :remove From 1ef3999c8f2035d4200b095f1fca9295839c0e48 Mon Sep 17 00:00:00 2001 From: vimesa Date: Fri, 24 May 2024 12:04:04 +0100 Subject: [PATCH 19/50] Release 1.9.0 --- CHANGELOG.md | 5 +++++ resources/metadata.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b77bb792..655cfe2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ cookbook-rb-manager CHANGELOG =============== + +## 1.9.0 + + - Miguel Negrón + - [ef94e3c] Add configure common cookbook call (#140) ## 1.8.0 - David Vanhoucke - [c2df76a] add postfix service diff --git a/resources/metadata.rb b/resources/metadata.rb index 2ee9625a..ad6bc700 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -4,7 +4,7 @@ license 'All rights reserved' description 'Installs/Configures redborder manager' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '1.8.0' +version '1.9.0' depends 'rb-common' depends 'chef-server' From cfe0e690f6d194a7e9a00c312797e9dfdbaaae8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Fri, 24 May 2024 12:34:53 +0100 Subject: [PATCH 20/50] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 655cfe2d..8ed24f77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ cookbook-rb-manager CHANGELOG - Miguel Negrón - [ef94e3c] Add configure common cookbook call (#140) + ## 1.8.0 - David Vanhoucke - [c2df76a] add postfix service From 31682a05a62f6e72aa28e239c17d28ef4fb6e7f7 Mon Sep 17 00:00:00 2001 From: Miguel Negron Date: Fri, 24 May 2024 22:32:04 +0100 Subject: [PATCH 21/50] Add full kernel release info in motd --- resources/templates/default/motd.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/templates/default/motd.erb b/resources/templates/default/motd.erb index f45def0d..58d805fd 100644 --- a/resources/templates/default/motd.erb +++ b/resources/templates/default/motd.erb @@ -3,7 +3,7 @@ Welcome to '<%= node["redborder"]["manager"]["cluster_name"] %>' redborder cluster: <% else %> Welcome to redborder-ng Manager [<%= node["hostname"] %>.<%= node["redborder"]["cdomain"] %>] (<%= node['platform'] %> - <%= node['platform_version'] %>): - Kernel: <%= node['os_version'].gsub("el9", "") %> + Kernel: <%= node['os_version'] %> <% end %> <% if !node["redborder"].nil? and !node["redborder"]["rpms"].nil? %> From 4a4efaa27c08246dde7d9453397015aa3965f89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Negr=C3=B3n?= <45871721+manegron@users.noreply.github.com> Date: Sun, 26 May 2024 09:49:14 +0100 Subject: [PATCH 22/50] Update metadata.rb --- resources/metadata.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/resources/metadata.rb b/resources/metadata.rb index ad6bc700..ac535a54 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -1,9 +1,8 @@ -name 'rb-manager' -maintainer 'Miguel Negrón' -maintainer_email 'manegron@redborder.com' -license 'All rights reserved' +name 'rb-manager' +maintainer 'Eneo Tecnología S.L.' +maintainer_email 'git@redborder.com' +license 'AGPL-3.0' description 'Installs/Configures redborder manager' -long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) version '1.9.0' depends 'rb-common' @@ -37,7 +36,6 @@ depends 'freeradius' depends 'rbcep' depends 'cron' -# depends 'ohai' depends 'rb-selinux' depends 'rbaioutliers' depends 'rbcgroup' From d29155fbe629b3817e682ffdc36cda2960b67125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Negr=C3=B3n?= <45871721+manegron@users.noreply.github.com> Date: Fri, 31 May 2024 00:57:24 +0100 Subject: [PATCH 23/50] Update rpm.yml --- .github/workflows/rpm.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rpm.yml b/.github/workflows/rpm.yml index 0672d54f..5d83b780 100644 --- a/.github/workflows/rpm.yml +++ b/.github/workflows/rpm.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - + - name: Create tag based on metadata.rb id: create_tag run: | @@ -26,41 +26,58 @@ jobs: echo "TAG=$TAG" >> $GITHUB_ENV shell: bash + - name: Check if Tag Exists + id: check_tag + run: | + if git rev-parse "refs/tags/${{ env.TAG }}" >/dev/null 2>&1; then + echo "Tag ${{ env.TAG }} already exists, exiting." + exit 1 + fi + shell: bash + - name: Set Version + if: success() run: echo "VERSION=${{ env.TAG }}" >> $GITHUB_ENV - name: Run Docker Container + if: success() run: docker run --privileged -d --name builder --network host rockylinux:9 /bin/sleep infinity - name: Install build tools RPM + if: success() run: | docker cp ./ builder:/build docker exec builder bash -c "yum install -y epel-release && yum install -y make git mock" docker exec builder bash -c "rm -rf /etc/mock/default.cfg" - name: Setup SDK + if: success() run: | docker exec builder bash -c "curl https://raw.githubusercontent.com/redBorder/repoinit/master/sdk9.cfg > /build/sdk9.cfg" docker exec builder bash -c "echo \"config_opts['use_host_resolv'] = True\" >> /build/sdk9.cfg" docker exec builder bash -c "ln -s /build/sdk9.cfg /etc/mock/default.cfg" - name: Build RPM using mock + if: success() run: | docker exec builder bash -c "git config --global --add safe.directory /build" docker exec builder bash -c "cd /build/ && VERSION=${{ env.TAG }} make rpm" - name: Copy RPMS + if: success() run: | docker cp builder:/build/packaging/rpm/pkgs/. ./rpms - name: Delete non-.rpm files + if: success() run: | find ./rpms -type f -not -name '*.rpm' -exec rm {} \; - name: Release + if: success() uses: softprops/action-gh-release@v1 with: files: ./rpms/* tag_name: ${{ env.TAG }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7d26b70af39781c85812d75c3ba6f85407693564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Negr=C3=B3n?= <45871721+manegron@users.noreply.github.com> Date: Fri, 31 May 2024 00:58:14 +0100 Subject: [PATCH 24/50] Update README.md --- README.md | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 42fc17dc..c77f0d16 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,25 @@ # manager Cookbook +[![Build Status][build-shield]][build-url] +[![Linters][linters-shield]][linters-url] +[![License][license-shield]][license-url] + + +[build-shield]: https://github.com/redBorder/cookbook-rb-manager/actions/workflows/rpm.yml/badge.svg?branch=master +[build-url]: https://github.com/redBorder/cookbook-rb-manager/actions/workflows/rpm.yml?query=branch%3Amaster +[linters-shield]: https://github.com/redBorder/cookbook-rb-manager/actions/workflows/lint.yml/badge.svg?event=push +[linters-url]: https://github.com/redBorder/cookbook-rb-manager/actions/workflows/lint.yml +[license-shield]: https://img.shields.io/badge/license-AGPLv3-blue.svg +[license-url]: https://github.com/cookbook-rb-manager/blob/HEAD/LICENSE Cookbook to configure a redborder manager -## Requirements - -depends 'kafka', '0.0.1' -depends 'zookeeper', '0.0.1' - ### Platforms - Rocky Linux 9 ### Chef -- Chef 12.0 or later - -# BUILDING - -- Build rpm package for redborder platform: - * git clone https://github.com/redborder/cookbook-rb-manager.git - * cd cookbook-rb-manager - * make - * RPM packages is under packaging/rpm/pkgs/ +- Chef 15.7.0 or later ## Contributing @@ -33,10 +31,5 @@ depends 'zookeeper', '0.0.1' 6. Submit a Pull Request using Github ## License -GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 -## Authors -Carlos Javier Mateos -Juan Jesús Prieto -Enrique Jimenez -Alberto Rodríguez +GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 From f22a469e5e1e6b91e00567c48f367473718d008e Mon Sep 17 00:00:00 2001 From: David Vanhoucke Date: Fri, 31 May 2024 14:33:34 +0100 Subject: [PATCH 25/50] add temporary variables in node.run_state --- resources/libraries/get_pipelines.rb | 2 +- resources/recipes/configure.rb | 58 ++++++++++++++-------------- resources/recipes/prepare_system.rb | 10 ++--- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/resources/libraries/get_pipelines.rb b/resources/libraries/get_pipelines.rb index c84920d6..f9fbeaa3 100644 --- a/resources/libraries/get_pipelines.rb +++ b/resources/libraries/get_pipelines.rb @@ -3,7 +3,7 @@ module Helpers def get_pipelines() logstash_pipelines = [] - sensors = node["redborder"]["sensors_info_all"] + sensors = node.run_state["sensors_info_all"] namespaces = get_namespaces() main_logstash = determine_main_logstash_node() diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 617840fc..8de0dc24 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -200,8 +200,8 @@ rbmonitor_config "Configure redborder-monitor" do name node["hostname"] - device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + device_nodes node.run_state["sensors_info_all"]["device-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] managers node["redborder"]["managers_list"] cluster node["redborder"]["cluster_info"] hostip node["redborder"]["cluster_info"][name]["ip"] @@ -209,7 +209,7 @@ end rbscanner_config "Configure redborder-scanner" do - scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] + scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] action (manager_services["redborder-scanner"] ? [:add, :register] : [:remove, :deregister]) end @@ -253,10 +253,10 @@ kafka_hosts node["redborder"]["managers_per_services"]["kafka"] memory node["redborder"]["memory_services"]["http2k"]["memory"] port node["redborder"]["http2k"]["port"] - proxy_nodes node["redborder"]["sensors_info"]["proxy-sensor"] - ips_nodes node["redborder"]["sensors_info"]["ips-sensor"] - ipsg_nodes node["redborder"]["sensors_info"]["ipsg-sensor"] - ipscp_nodes node["redborder"]["sensors_info"]["ipscp-sensor"] + proxy_nodes node.run_state["sensors_info"]["proxy-sensor"] + ips_nodes node.run_state["sensors_info"]["ips-sensor"] + ipsg_nodes node.run_state["sensors_info"]["ipsg-sensor"] + ipscp_nodes node.run_state["sensors_info"]["ipscp-sensor"] organizations node["redborder"]["organizations"] locations_list node["redborder"]["locations"] action (manager_services["http2k"] ? [:add, :register] : [:remove, :deregister]) @@ -269,29 +269,29 @@ end f2k_config "Configure f2k" do - sensors node["redborder"]["sensors_info"]["flow-sensor"] + sensors node.run_state["sensors_info"]["flow-sensor"] action (manager_services["f2k"] ? [:add, :register] : [:remove, :deregister]) end pmacct_config "Configure pmacct" do - sensors node["redborder"]["sensors_info"]["flow-sensor"] + sensors node.run_state["sensors_info"]["flow-sensor"] kafka_hosts node["redborder"]["managers_per_services"]["kafka"] action (manager_services["pmacct"] ? [:add, :register] : [:remove, :deregister]) end -if node["redborder"]["logstash"]["pipelines"].nil? || node["redborder"]["logstash"]["pipelines"].empty? +if node.run_state["pipelines"].nil? || node.run_state["pipelines"].empty? service 'logstash' do action [:disable, :stop] end else logstash_config "Configure logstash" do cdomain node["redborder"]["cdomain"] - flow_nodes node["redborder"]["all_flow_sensors_info"]["flow-sensor"] - namespaces node["redborder"]["namespaces"] - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] - scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] - device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] - logstash_pipelines node["redborder"]["logstash"]["pipelines"] + flow_nodes node.run_state["all_flow_sensors_info"]["flow-sensor"] + namespaces node.run_state["namespaces"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] + device_nodes node.run_state["sensors_info_all"]["device-sensor"] + logstash_pipelines node.run_state["pipelines"] action (manager_services["logstash"] ? [:add, :register] : [:remove, :deregister]) end end @@ -307,29 +307,29 @@ end rsyslog_config "Configure rsyslog" do - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] + node["redborder"]["sensors_info_all"]["cep-sensor"] - ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + node.run_state["sensors_info_all"]["cep-sensor"] + ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] action (manager_services["rsyslog"] ? [:add, :register] : [:remove, :deregister]) end rbnmsp_config "Configure redborder-nmsp" do memory node["redborder"]["memory_services"]["redborder-nmsp"]["memory"] - proxy_nodes node["redborder"]["sensors_info_all"]["proxy-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + proxy_nodes node.run_state["sensors_info_all"]["proxy-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] hosts node["redborder"]["zookeeper"]["zk_hosts"] action (manager_services["redborder-nmsp"] ? [:add, :configure_keys, :register] : [:remove, :deregister]) end n2klocd_config "Configure n2klocd" do - mse_nodes node["redborder"]["sensors_info_all"]["mse-sensor"] - meraki_nodes node["redborder"]["sensors_info_all"]["meraki-sensor"] + mse_nodes node.run_state["sensors_info_all"]["mse-sensor"] + meraki_nodes node.run_state["sensors_info_all"]["meraki-sensor"] n2klocd_managers node["redborder"]["managers_per_services"]["n2klocd"] memory node["redborder"]["memory_services"]["n2klocd"]["memory"] action (manager_services["n2klocd"] ? [:add, :register] : [:remove, :deregister]) end rbale_config "Configure redborder-ale" do - ale_nodes node["redborder"]["sensors_info_all"]["ale-sensor"] + ale_nodes node.run_state["sensors_info_all"]["ale-sensor"] action (node["redborder"]["services"]["redborder-ale"] ? [:add, :register] : [:remove, :deregister]) end @@ -338,13 +338,13 @@ end rb_arubacentral_config "Configure rb-arubacentral" do - arubacentral_nodes node["redborder"]["sensors_info_all"]["arubacentral-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + arubacentral_nodes node.run_state["sensors_info_all"]["arubacentral-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] action (node["redborder"]["services"]["rb-arubacentral"] ? :add : :remove) end #freeradius_config "Configure radiusd" do -# flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] +# flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] # action (node["redborder"]["services"]["radiusd"] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) #end @@ -353,9 +353,9 @@ end rbcep_config "Configure redborder-cep" do - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] - ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] action (node["redborder"]["services"]["redborder-cep"] ? [:add, :register] : [:remove, :deregister]) end diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 0f97e046..7551a7d6 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -87,19 +87,19 @@ node.default["redborder"]["organizations"] = get_orgs() if node["redborder"]["services"]["http2k"] #get sensors info -node.default["redborder"]["sensors_info"] = get_sensors_info() +node.run_state["sensors_info"] = get_sensors_info() #get sensors info full info -node.default["redborder"]["sensors_info_all"] = get_sensors_all_info() +node.run_state["sensors_info_all"] = get_sensors_all_info() #get sensors info of all flow sensors -node.default["redborder"]["all_flow_sensors_info"] = get_all_flow_sensors_info() +node.run_state["all_flow_sensors_info"] = get_all_flow_sensors_info() #get logstash pipelines -node.default["redborder"]["logstash"]["pipelines"] = get_pipelines() +node.run_state["pipelines"] = get_pipelines() #get namespaces -node.default["redborder"]["namespaces"] = get_namespaces +node.run_state["namespaces"] = get_namespaces #get string with all zookeeper hosts and port separated by commas, its needed for multiples services zk_port = node["redborder"]["zookeeper"]["port"] From f9a20f7d3150d3c872a3d27796805c0fe5b6f367 Mon Sep 17 00:00:00 2001 From: JuanSheba Date: Mon, 3 Jun 2024 10:00:33 +0100 Subject: [PATCH 26/50] Release 1.9.2 --- CHANGELOG.md | 11 +++++++++++ resources/metadata.rb | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed24f77..8671b2c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ cookbook-rb-manager CHANGELOG =============== +## 1.9.1 + + - David Vanhoucke + - [f22a469] add temporary variables in node.run_state + - Miguel Negrón + - [7d26b70] Update README.md + - [d29155f] Update rpm.yml + - [4a4efaa] Update metadata.rb + - [1120885] Merge pull request #142 from redBorder/bugfix/missing_specific_dist_kernel_info_in_motd + - [31682a0] Add full kernel release info in motd + ## 1.9.0 - Miguel Negrón diff --git a/resources/metadata.rb b/resources/metadata.rb index ac535a54..090183cb 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,7 +3,7 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures redborder manager' -version '1.9.0' +version '1.9.1' depends 'rb-common' depends 'chef-server' From 1ecea6915982a2c01a149a14a3992077999afe02 Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:46:57 +0100 Subject: [PATCH 27/50] Update rpm.yml remove whitespaces of empty line --- .github/workflows/rpm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rpm.yml b/.github/workflows/rpm.yml index 5d83b780..73dd7280 100644 --- a/.github/workflows/rpm.yml +++ b/.github/workflows/rpm.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - + - name: Create tag based on metadata.rb id: create_tag run: | From f08e447f7e2dd7b406a7afd48c3765d3a303c104 Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Mon, 3 Jun 2024 16:53:15 +0100 Subject: [PATCH 28/50] Configure minio nodes and load balancer --- resources/attributes/default.rb | 1 + resources/recipes/configure.rb | 72 +++++++++++++++------------ resources/recipes/prepare_system.rb | 14 ++++-- resources/templates/default/minio.erb | 4 ++ 4 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 resources/templates/default/minio.erb diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 2a54bfaf..db1ef8fc 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -13,6 +13,7 @@ #s3 default["redborder"]["uploaded_s3"] = false +default["redborder"]["s3"]["s3_hosts"] = [] #chef-client default["chef-client"]["interval"] = 300 diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 8de0dc24..289ae9c6 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -200,8 +200,8 @@ rbmonitor_config "Configure redborder-monitor" do name node["hostname"] - device_nodes node.run_state["sensors_info_all"]["device-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] + device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] + flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] managers node["redborder"]["managers_list"] cluster node["redborder"]["cluster_info"] hostip node["redborder"]["cluster_info"][name]["ip"] @@ -209,7 +209,7 @@ end rbscanner_config "Configure redborder-scanner" do - scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] + scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] action (manager_services["redborder-scanner"] ? [:add, :register] : [:remove, :deregister]) end @@ -253,10 +253,10 @@ kafka_hosts node["redborder"]["managers_per_services"]["kafka"] memory node["redborder"]["memory_services"]["http2k"]["memory"] port node["redborder"]["http2k"]["port"] - proxy_nodes node.run_state["sensors_info"]["proxy-sensor"] - ips_nodes node.run_state["sensors_info"]["ips-sensor"] - ipsg_nodes node.run_state["sensors_info"]["ipsg-sensor"] - ipscp_nodes node.run_state["sensors_info"]["ipscp-sensor"] + proxy_nodes node["redborder"]["sensors_info"]["proxy-sensor"] + ips_nodes node["redborder"]["sensors_info"]["ips-sensor"] + ipsg_nodes node["redborder"]["sensors_info"]["ipsg-sensor"] + ipscp_nodes node["redborder"]["sensors_info"]["ipscp-sensor"] organizations node["redborder"]["organizations"] locations_list node["redborder"]["locations"] action (manager_services["http2k"] ? [:add, :register] : [:remove, :deregister]) @@ -269,29 +269,29 @@ end f2k_config "Configure f2k" do - sensors node.run_state["sensors_info"]["flow-sensor"] + sensors node["redborder"]["sensors_info"]["flow-sensor"] action (manager_services["f2k"] ? [:add, :register] : [:remove, :deregister]) end pmacct_config "Configure pmacct" do - sensors node.run_state["sensors_info"]["flow-sensor"] + sensors node["redborder"]["sensors_info"]["flow-sensor"] kafka_hosts node["redborder"]["managers_per_services"]["kafka"] action (manager_services["pmacct"] ? [:add, :register] : [:remove, :deregister]) end -if node.run_state["pipelines"].nil? || node.run_state["pipelines"].empty? +if node["redborder"]["logstash"]["pipelines"].nil? || node["redborder"]["logstash"]["pipelines"].empty? service 'logstash' do action [:disable, :stop] end else logstash_config "Configure logstash" do cdomain node["redborder"]["cdomain"] - flow_nodes node.run_state["all_flow_sensors_info"]["flow-sensor"] - namespaces node.run_state["namespaces"] - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] - scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] - device_nodes node.run_state["sensors_info_all"]["device-sensor"] - logstash_pipelines node.run_state["pipelines"] + flow_nodes node["redborder"]["all_flow_sensors_info"]["flow-sensor"] + namespaces node["redborder"]["namespaces"] + vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] + scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] + device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] + logstash_pipelines node["redborder"]["logstash"]["pipelines"] action (manager_services["logstash"] ? [:add, :register] : [:remove, :deregister]) end end @@ -307,29 +307,29 @@ end rsyslog_config "Configure rsyslog" do - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + node.run_state["sensors_info_all"]["cep-sensor"] - ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] + vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] + node["redborder"]["sensors_info_all"]["cep-sensor"] + ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] action (manager_services["rsyslog"] ? [:add, :register] : [:remove, :deregister]) end rbnmsp_config "Configure redborder-nmsp" do memory node["redborder"]["memory_services"]["redborder-nmsp"]["memory"] - proxy_nodes node.run_state["sensors_info_all"]["proxy-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] + proxy_nodes node["redborder"]["sensors_info_all"]["proxy-sensor"] + flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] hosts node["redborder"]["zookeeper"]["zk_hosts"] action (manager_services["redborder-nmsp"] ? [:add, :configure_keys, :register] : [:remove, :deregister]) end n2klocd_config "Configure n2klocd" do - mse_nodes node.run_state["sensors_info_all"]["mse-sensor"] - meraki_nodes node.run_state["sensors_info_all"]["meraki-sensor"] + mse_nodes node["redborder"]["sensors_info_all"]["mse-sensor"] + meraki_nodes node["redborder"]["sensors_info_all"]["meraki-sensor"] n2klocd_managers node["redborder"]["managers_per_services"]["n2klocd"] memory node["redborder"]["memory_services"]["n2klocd"]["memory"] action (manager_services["n2klocd"] ? [:add, :register] : [:remove, :deregister]) end rbale_config "Configure redborder-ale" do - ale_nodes node.run_state["sensors_info_all"]["ale-sensor"] + ale_nodes node["redborder"]["sensors_info_all"]["ale-sensor"] action (node["redborder"]["services"]["redborder-ale"] ? [:add, :register] : [:remove, :deregister]) end @@ -338,13 +338,13 @@ end rb_arubacentral_config "Configure rb-arubacentral" do - arubacentral_nodes node.run_state["sensors_info_all"]["arubacentral-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] + arubacentral_nodes node["redborder"]["sensors_info_all"]["arubacentral-sensor"] + flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] action (node["redborder"]["services"]["rb-arubacentral"] ? :add : :remove) end #freeradius_config "Configure radiusd" do -# flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] +# flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] # action (node["redborder"]["services"]["radiusd"] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) #end @@ -353,9 +353,9 @@ end rbcep_config "Configure redborder-cep" do - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] - ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] + flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] + ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] action (node["redborder"]["services"]["redborder-cep"] ? [:add, :register] : [:remove, :deregister]) end @@ -376,12 +376,20 @@ action (manager_services["postgresql"] and external_services["postgresql"] == "onpremise" ? [:add, :register] : [:remove, :deregister]) end -s3_leader = `serf members | grep s3=ready | awk '{print $1'} | head -n 1`.strip +s3_secrets = Chef::DataBagItem.load('passwords','s3') -# Allow only one s3 onpremise node for now.. TODO: Distributed MinIO +# Allow only s3 onpremise nodes for now.. minio_config "Configure S3 (minio)" do ipaddress node["ipaddress_sync"] - action ((manager_services["s3"] and external_services["s3"] == "onpremise" and s3_leader == node.name ) ? [:add, :register] : [:remove, :deregister]) + access_key_id s3_secrets["s3_access_key_id"] + secret_key_id s3_secrets["s3_secret_key_id"] + action ((manager_services["s3"] and external_services["s3"] == "onpremise") ? [:add, :register] : [:remove, :deregister]) +end + +# Configure Nginx s3 onpremise nodes for now.. +minio_config "Configure Nginx S3 (minio)" do + s3_hosts node["redborder"]["s3_hosts"] + action ((manager_services["s3"] and external_services["s3"] == "onpremise") ? [:add_s3_conf_nginx] : :nothing) end if manager_services["s3"] diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 7551a7d6..0f32bf8d 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -87,25 +87,29 @@ node.default["redborder"]["organizations"] = get_orgs() if node["redborder"]["services"]["http2k"] #get sensors info -node.run_state["sensors_info"] = get_sensors_info() +node.default["redborder"]["sensors_info"] = get_sensors_info() #get sensors info full info -node.run_state["sensors_info_all"] = get_sensors_all_info() +node.default["redborder"]["sensors_info_all"] = get_sensors_all_info() #get sensors info of all flow sensors -node.run_state["all_flow_sensors_info"] = get_all_flow_sensors_info() +node.default["redborder"]["all_flow_sensors_info"] = get_all_flow_sensors_info() #get logstash pipelines -node.run_state["pipelines"] = get_pipelines() +node.default["redborder"]["logstash"]["pipelines"] = get_pipelines() #get namespaces -node.run_state["namespaces"] = get_namespaces +node.default["redborder"]["namespaces"] = get_namespaces #get string with all zookeeper hosts and port separated by commas, its needed for multiples services zk_port = node["redborder"]["zookeeper"]["port"] #zk_hosts = node["redborder"]["managers_per_services"]["zookeeper"].map {|z| "#{z}.node:#{zk_port}"}.join(',') node.default["redborder"]["zookeeper"]["zk_hosts"] = "zookeeper.service.#{node["redborder"]["cdomain"]}:#{node["redborder"]["zookeeper"]["port"]}" +# Set all nodes with s3 configured (nginx load balancer) +s3_hosts = node["redborder"]["managers_per_services"]["s3"].map {|z| "#{z}.node:9000"} +node.default["redborder"]["s3"]["s3_hosts"] = s3_hosts + #set kafka host index if kafka is enabled in this host if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) node.default["redborder"]["kafka"]["host_index"] = node["redborder"]["managers_per_services"]["kafka"].index(node.name) diff --git a/resources/templates/default/minio.erb b/resources/templates/default/minio.erb new file mode 100644 index 00000000..4973ec55 --- /dev/null +++ b/resources/templates/default/minio.erb @@ -0,0 +1,4 @@ +MINIO_OPTS="--address :9000 --console-address :9001 --config-dir /etc/minio" +MINIO_VOLUMES=/var/minio/data +MINIO_ROOT_USER=<%= @s3_user %> +MINIO_ROOT_PASSWORD=<%= @s3_password %> \ No newline at end of file From ac54c02592ce9000af4b61c3bba3e713dbc81cbc Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Mon, 3 Jun 2024 16:54:51 +0100 Subject: [PATCH 29/50] Proper use of s3 hosts --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 289ae9c6..5b6e7785 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -388,7 +388,7 @@ # Configure Nginx s3 onpremise nodes for now.. minio_config "Configure Nginx S3 (minio)" do - s3_hosts node["redborder"]["s3_hosts"] + s3_hosts node["redborder"]["s3"]["s3_hosts"] action ((manager_services["s3"] and external_services["s3"] == "onpremise") ? [:add_s3_conf_nginx] : :nothing) end From fd63ad2e46a01574eb1ea2bd1a5929f46a8ffcf7 Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Mon, 3 Jun 2024 16:57:24 +0100 Subject: [PATCH 30/50] Fix prepare and configure --- resources/recipes/configure.rb | 58 ++++++++++++++--------------- resources/recipes/prepare_system.rb | 10 ++--- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 5b6e7785..924a0f3f 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -200,8 +200,8 @@ rbmonitor_config "Configure redborder-monitor" do name node["hostname"] - device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + device_nodes node.run_state["sensors_info_all"]["device-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] managers node["redborder"]["managers_list"] cluster node["redborder"]["cluster_info"] hostip node["redborder"]["cluster_info"][name]["ip"] @@ -209,7 +209,7 @@ end rbscanner_config "Configure redborder-scanner" do - scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] + scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] action (manager_services["redborder-scanner"] ? [:add, :register] : [:remove, :deregister]) end @@ -253,10 +253,10 @@ kafka_hosts node["redborder"]["managers_per_services"]["kafka"] memory node["redborder"]["memory_services"]["http2k"]["memory"] port node["redborder"]["http2k"]["port"] - proxy_nodes node["redborder"]["sensors_info"]["proxy-sensor"] - ips_nodes node["redborder"]["sensors_info"]["ips-sensor"] - ipsg_nodes node["redborder"]["sensors_info"]["ipsg-sensor"] - ipscp_nodes node["redborder"]["sensors_info"]["ipscp-sensor"] + proxy_nodes node.run_state["sensors_info"]["proxy-sensor"] + ips_nodes node.run_state["sensors_info"]["ips-sensor"] + ipsg_nodes node.run_state["sensors_info"]["ipsg-sensor"] + ipscp_nodes node.run_state["sensors_info"]["ipscp-sensor"] organizations node["redborder"]["organizations"] locations_list node["redborder"]["locations"] action (manager_services["http2k"] ? [:add, :register] : [:remove, :deregister]) @@ -269,29 +269,29 @@ end f2k_config "Configure f2k" do - sensors node["redborder"]["sensors_info"]["flow-sensor"] + sensors node.run_state["sensors_info"]["flow-sensor"] action (manager_services["f2k"] ? [:add, :register] : [:remove, :deregister]) end pmacct_config "Configure pmacct" do - sensors node["redborder"]["sensors_info"]["flow-sensor"] + sensors node.run_state["sensors_info"]["flow-sensor"] kafka_hosts node["redborder"]["managers_per_services"]["kafka"] action (manager_services["pmacct"] ? [:add, :register] : [:remove, :deregister]) end -if node["redborder"]["logstash"]["pipelines"].nil? || node["redborder"]["logstash"]["pipelines"].empty? +if node.run_state["pipelines"].nil? || node.run_state["pipelines"].empty? service 'logstash' do action [:disable, :stop] end else logstash_config "Configure logstash" do cdomain node["redborder"]["cdomain"] - flow_nodes node["redborder"]["all_flow_sensors_info"]["flow-sensor"] - namespaces node["redborder"]["namespaces"] - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] - scanner_nodes node["redborder"]["sensors_info_all"]["scanner-sensor"] - device_nodes node["redborder"]["sensors_info_all"]["device-sensor"] - logstash_pipelines node["redborder"]["logstash"]["pipelines"] + flow_nodes node.run_state["all_flow_sensors_info"]["flow-sensor"] + namespaces node.run_state["namespaces"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] + device_nodes node.run_state["sensors_info_all"]["device-sensor"] + logstash_pipelines node.run_state["pipelines"] action (manager_services["logstash"] ? [:add, :register] : [:remove, :deregister]) end end @@ -307,29 +307,29 @@ end rsyslog_config "Configure rsyslog" do - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] + node["redborder"]["sensors_info_all"]["cep-sensor"] - ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + node.run_state["sensors_info_all"]["cep-sensor"] + ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] action (manager_services["rsyslog"] ? [:add, :register] : [:remove, :deregister]) end rbnmsp_config "Configure redborder-nmsp" do memory node["redborder"]["memory_services"]["redborder-nmsp"]["memory"] - proxy_nodes node["redborder"]["sensors_info_all"]["proxy-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + proxy_nodes node.run_state["sensors_info_all"]["proxy-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] hosts node["redborder"]["zookeeper"]["zk_hosts"] action (manager_services["redborder-nmsp"] ? [:add, :configure_keys, :register] : [:remove, :deregister]) end n2klocd_config "Configure n2klocd" do - mse_nodes node["redborder"]["sensors_info_all"]["mse-sensor"] - meraki_nodes node["redborder"]["sensors_info_all"]["meraki-sensor"] + mse_nodes node.run_state["sensors_info_all"]["mse-sensor"] + meraki_nodes node.run_state["sensors_info_all"]["meraki-sensor"] n2klocd_managers node["redborder"]["managers_per_services"]["n2klocd"] memory node["redborder"]["memory_services"]["n2klocd"]["memory"] action (manager_services["n2klocd"] ? [:add, :register] : [:remove, :deregister]) end rbale_config "Configure redborder-ale" do - ale_nodes node["redborder"]["sensors_info_all"]["ale-sensor"] + ale_nodes node.run_state["sensors_info_all"]["ale-sensor"] action (node["redborder"]["services"]["redborder-ale"] ? [:add, :register] : [:remove, :deregister]) end @@ -338,13 +338,13 @@ end rb_arubacentral_config "Configure rb-arubacentral" do - arubacentral_nodes node["redborder"]["sensors_info_all"]["arubacentral-sensor"] - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] + arubacentral_nodes node.run_state["sensors_info_all"]["arubacentral-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] action (node["redborder"]["services"]["rb-arubacentral"] ? :add : :remove) end #freeradius_config "Configure radiusd" do -# flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] +# flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] # action (node["redborder"]["services"]["radiusd"] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) #end @@ -353,9 +353,9 @@ end rbcep_config "Configure redborder-cep" do - flow_nodes node["redborder"]["sensors_info_all"]["flow-sensor"] - vault_nodes node["redborder"]["sensors_info_all"]["vault-sensor"] - ips_nodes node["redborder"]["sensors_info_all"]["ips-sensor"] + node["redborder"]["sensors_info_all"]["ipsv2-sensor"] + node["redborder"]["sensors_info_all"]["ipscp-sensor"] + flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] + vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] action (node["redborder"]["services"]["redborder-cep"] ? [:add, :register] : [:remove, :deregister]) end diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 0f32bf8d..0b403d12 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -87,19 +87,19 @@ node.default["redborder"]["organizations"] = get_orgs() if node["redborder"]["services"]["http2k"] #get sensors info -node.default["redborder"]["sensors_info"] = get_sensors_info() +node.run_state["sensors_info"] = get_sensors_info() #get sensors info full info -node.default["redborder"]["sensors_info_all"] = get_sensors_all_info() +node.run_state["sensors_info_all"] = get_sensors_all_info() #get sensors info of all flow sensors -node.default["redborder"]["all_flow_sensors_info"] = get_all_flow_sensors_info() +node.run_state["all_flow_sensors_info"] = get_all_flow_sensors_info() #get logstash pipelines -node.default["redborder"]["logstash"]["pipelines"] = get_pipelines() +node.run_state["pipelines"] = get_pipelines() #get namespaces -node.default["redborder"]["namespaces"] = get_namespaces +node.run_state["namespaces"] = get_namespaces #get string with all zookeeper hosts and port separated by commas, its needed for multiples services zk_port = node["redborder"]["zookeeper"]["port"] From 65696efd626f44f38911f9a52de592f8c69ec7d9 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Thu, 6 Jun 2024 09:32:37 +0100 Subject: [PATCH 31/50] configure not removing geoip. Geoip won't be in the list because is no longer a service --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 8de0dc24..579301ee 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -189,7 +189,7 @@ end geoip_config "Configure GeoIP" do - action (manager_services["geoip"] ? :add : :remove) + action :add end snmp_config "Configure snmp" do From 87934304bd5f3ddf3fc4ac72435bbd645be4236e Mon Sep 17 00:00:00 2001 From: Miguel Negron Date: Fri, 7 Jun 2024 12:47:27 +0100 Subject: [PATCH 32/50] Bump version --- CHANGELOG.md | 5 +++++ resources/metadata.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8671b2c6..21f96a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ cookbook-rb-manager CHANGELOG =============== +## 1.9.2 + + - Miguel Negrón + - [65696ef] configure not removing geoip. Geoip wont be in the list because is no longer a service + ## 1.9.1 - David Vanhoucke diff --git a/resources/metadata.rb b/resources/metadata.rb index 090183cb..3ce5b42e 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,7 +3,7 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures redborder manager' -version '1.9.1' +version '1.9.2' depends 'rb-common' depends 'chef-server' From 292288d42c7d56bfc1459832d89cda7f44524859 Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Mon, 10 Jun 2024 13:53:14 +0100 Subject: [PATCH 33/50] Add hosts in node data --- resources/attributes/default.rb | 1 + resources/recipes/configure.rb | 1 + resources/recipes/prepare_system.rb | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index db1ef8fc..708321ac 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -34,6 +34,7 @@ #webui default["redborder"]["webui"]["port"] = 8001 +default["redborder"]["webui"]["hosts"] = [] #memcached default["redborder"]["memcached"]["elasticache"] = false diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 924a0f3f..c11965b9 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -243,6 +243,7 @@ end webui_config "Configure Nginx WebUI" do + hosts node["redborder"]["webui"]["webui_hosts"] cdomain node["redborder"]["cdomain"] port node["redborder"]["webui"]["port"] action ((manager_services["webui"] and manager_services["nginx"]) ? [:configure_certs, :add_webui_conf_nginx] : :nothing) diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 0b403d12..6a414bd2 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -110,6 +110,11 @@ s3_hosts = node["redborder"]["managers_per_services"]["s3"].map {|z| "#{z}.node:9000"} node.default["redborder"]["s3"]["s3_hosts"] = s3_hosts +# set webui hosts +webui_hosts = node["redborder"]["managers_per_services"]["webui"].map {|z| "#{z}.node"} +node.default["redborder"]["webui"]["webui_hosts"] = webui_hosts + + #set kafka host index if kafka is enabled in this host if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) node.default["redborder"]["kafka"]["host_index"] = node["redborder"]["managers_per_services"]["kafka"].index(node.name) From 6e7680a3eaaa6850334bcd3c13298f43dd0fb924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:58:26 +0100 Subject: [PATCH 34/50] Delete resources/templates/default/minio.erb --- resources/templates/default/minio.erb | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 resources/templates/default/minio.erb diff --git a/resources/templates/default/minio.erb b/resources/templates/default/minio.erb deleted file mode 100644 index 4973ec55..00000000 --- a/resources/templates/default/minio.erb +++ /dev/null @@ -1,4 +0,0 @@ -MINIO_OPTS="--address :9000 --console-address :9001 --config-dir /etc/minio" -MINIO_VOLUMES=/var/minio/data -MINIO_ROOT_USER=<%= @s3_user %> -MINIO_ROOT_PASSWORD=<%= @s3_password %> \ No newline at end of file From 3bc8e8f7d3700c7ab607612a13fcdcbd2ad2e4e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:58:45 +0100 Subject: [PATCH 35/50] Update prepare_system.rb --- resources/recipes/prepare_system.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 6a414bd2..ac6ab0c1 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -106,10 +106,6 @@ #zk_hosts = node["redborder"]["managers_per_services"]["zookeeper"].map {|z| "#{z}.node:#{zk_port}"}.join(',') node.default["redborder"]["zookeeper"]["zk_hosts"] = "zookeeper.service.#{node["redborder"]["cdomain"]}:#{node["redborder"]["zookeeper"]["port"]}" -# Set all nodes with s3 configured (nginx load balancer) -s3_hosts = node["redborder"]["managers_per_services"]["s3"].map {|z| "#{z}.node:9000"} -node.default["redborder"]["s3"]["s3_hosts"] = s3_hosts - # set webui hosts webui_hosts = node["redborder"]["managers_per_services"]["webui"].map {|z| "#{z}.node"} node.default["redborder"]["webui"]["webui_hosts"] = webui_hosts From ed91ff3faadc1333a3872af33bdb1ab70cd0b178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:59:03 +0100 Subject: [PATCH 36/50] Update default.rb --- resources/attributes/default.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 708321ac..8d5faeb5 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -13,7 +13,6 @@ #s3 default["redborder"]["uploaded_s3"] = false -default["redborder"]["s3"]["s3_hosts"] = [] #chef-client default["chef-client"]["interval"] = 300 From d89bd26f0eff98a2f623c7572141731d733566b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:00:07 +0100 Subject: [PATCH 37/50] Update configure.rb --- resources/recipes/configure.rb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index c11965b9..1e7a5250 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -382,15 +382,7 @@ # Allow only s3 onpremise nodes for now.. minio_config "Configure S3 (minio)" do ipaddress node["ipaddress_sync"] - access_key_id s3_secrets["s3_access_key_id"] - secret_key_id s3_secrets["s3_secret_key_id"] - action ((manager_services["s3"] and external_services["s3"] == "onpremise") ? [:add, :register] : [:remove, :deregister]) -end - -# Configure Nginx s3 onpremise nodes for now.. -minio_config "Configure Nginx S3 (minio)" do - s3_hosts node["redborder"]["s3"]["s3_hosts"] - action ((manager_services["s3"] and external_services["s3"] == "onpremise") ? [:add_s3_conf_nginx] : :nothing) + action ((manager_services["s3"] and external_services["s3"] == "onpremise" and s3_leader == node.name ) ? [:add, :register] : [:remove, :deregister]) end if manager_services["s3"] From 4ec77f8f74a508c56577912aa75d1f414222974c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:00:35 +0100 Subject: [PATCH 38/50] Update configure.rb --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 1e7a5250..07e14e73 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -377,7 +377,7 @@ action (manager_services["postgresql"] and external_services["postgresql"] == "onpremise" ? [:add, :register] : [:remove, :deregister]) end -s3_secrets = Chef::DataBagItem.load('passwords','s3') +s3_leader = `serf members | grep s3=ready | awk '{print $1'} | head -n 1`.strip # Allow only s3 onpremise nodes for now.. minio_config "Configure S3 (minio)" do From 3220e71722ca01f16b9c672c861354a44d5b9f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:01:02 +0100 Subject: [PATCH 39/50] Update configure.rb --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 07e14e73..bbed9d6b 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -379,7 +379,7 @@ s3_leader = `serf members | grep s3=ready | awk '{print $1'} | head -n 1`.strip -# Allow only s3 onpremise nodes for now.. +# Allow only one s3 onpremise node for now.. TODO: Distributed MinIO minio_config "Configure S3 (minio)" do ipaddress node["ipaddress_sync"] action ((manager_services["s3"] and external_services["s3"] == "onpremise" and s3_leader == node.name ) ? [:add, :register] : [:remove, :deregister]) From e101c2daf7d11e59febf41daf856098f166bc085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:01:33 +0100 Subject: [PATCH 40/50] Update prepare_system.rb --- resources/recipes/prepare_system.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index ac6ab0c1..8e80f2a9 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -110,7 +110,6 @@ webui_hosts = node["redborder"]["managers_per_services"]["webui"].map {|z| "#{z}.node"} node.default["redborder"]["webui"]["webui_hosts"] = webui_hosts - #set kafka host index if kafka is enabled in this host if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) node.default["redborder"]["kafka"]["host_index"] = node["redborder"]["managers_per_services"]["kafka"].index(node.name) From 4db8fa127f35ff35df6b25dfcb309c1f26c8aab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:12:41 +0100 Subject: [PATCH 41/50] Update configure.rb --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index bbed9d6b..95db2925 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -243,7 +243,7 @@ end webui_config "Configure Nginx WebUI" do - hosts node["redborder"]["webui"]["webui_hosts"] + hosts node["redborder"]["webui"]["hosts"] cdomain node["redborder"]["cdomain"] port node["redborder"]["webui"]["port"] action ((manager_services["webui"] and manager_services["nginx"]) ? [:configure_certs, :add_webui_conf_nginx] : :nothing) From 78f1562db7a62fa9cc39a774f6bfe3dbf439a024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:13:02 +0100 Subject: [PATCH 42/50] Update prepare_system.rb --- resources/recipes/prepare_system.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 8e80f2a9..2da1dfe4 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -108,7 +108,7 @@ # set webui hosts webui_hosts = node["redborder"]["managers_per_services"]["webui"].map {|z| "#{z}.node"} -node.default["redborder"]["webui"]["webui_hosts"] = webui_hosts +node.default["redborder"]["webui"]["hosts"] = webui_hosts #set kafka host index if kafka is enabled in this host if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) From a12be850bb9ca2268de4c9bb9318b1dbedf3634b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Negr=C3=B3n?= <45871721+manegron@users.noreply.github.com> Date: Tue, 11 Jun 2024 08:55:17 +0100 Subject: [PATCH 43/50] Improvement/fix lint (#148) * Fix: Update styles in attributes * Fix: Update styles in libraries * Fix: Update styles in recipes * Fix: Rename template .rb to .rb.erb * Fix: typo in configure.rb * Fix: rework the way we get ipaddress_sync * Fix: retrieve sync ip * Fix: title of README --- README.md | 2 +- resources/attributes/default.rb | 390 +++++---- .../libraries/get_all_flow_sensors_info.rb | 13 +- resources/libraries/get_cluster_info.rb | 33 +- resources/libraries/get_elasticache_nodes.rb | 18 +- resources/libraries/get_managers_all.rb | 15 +- resources/libraries/get_namespaces.rb | 13 +- resources/libraries/get_orgs.rb | 15 +- resources/libraries/get_pipelines.rb | 43 +- resources/libraries/get_sensors_all_info.rb | 13 +- resources/libraries/get_sensors_info.rb | 37 +- resources/libraries/harddisk_services.rb | 113 +-- resources/libraries/in_hash.rb | 10 +- resources/libraries/manager_services.rb | 28 +- resources/libraries/managers_per_service.rb | 15 +- resources/libraries/memory_services.rb | 61 +- resources/libraries/node_services.rb | 26 +- resources/libraries/util.rb | 13 +- resources/recipes/configure.rb | 743 +++++++++++------- resources/recipes/configure_cron_tasks.rb | 57 +- resources/recipes/configure_journald.rb | 37 +- resources/recipes/default.rb | 10 +- resources/recipes/prepare_system.rb | 188 +++-- ...client.rb => sysconfig_chef-client.rb.erb} | 0 24 files changed, 1013 insertions(+), 880 deletions(-) rename resources/templates/default/{sysconfig_chef-client.rb => sysconfig_chef-client.rb.erb} (100%) diff --git a/README.md b/README.md index c77f0d16..fe8240e4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# manager Cookbook +# cookbook-rb-manager [![Build Status][build-shield]][build-url] [![Linters][linters-shield]][linters-url] [![License][license-shield]][license-url] diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 2a54bfaf..ff2f11b4 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -1,222 +1,214 @@ -#Default attributes - -#general -default["redborder"]["cdomain"] = "redborder.cluster" -default["redborder"]["organization_uuid"] = nil -default["redborder"]["organizations"] = [] -default["redborder"]["locations"] = [ - "namespace", "namespace_uuid", "organization", "organization_uuid", \ - "service_provider", "service_provider_uuid", "deployment", \ - "deployment_uuid", "market", "market_uuid", "campus", "campus_uuid", \ - "building", "building_uuid", "floor", "floor_uuid" - ] - -#s3 -default["redborder"]["uploaded_s3"] = false - -#chef-client -default["chef-client"]["interval"] = 300 -default["chef-client"]["splay"] = 100 -default["chef-client"]["options"] = "" - -#kafka -default["redborder"]["kafka"]["port"] = 9092 -default["redborder"]["kafka"]["logdir"] = "/var/log/kafka" -default["redborder"]["kafka"]["host_index"] = 0 - -#zookeeper -default["redborder"]["zookeeper"]["zk_hosts"] = "" -default["redborder"]["zookeeper"]["port"] = 2181 - -#http2k -default["redborder"]["http2k"]["port"] = 7980 - -#webui -default["redborder"]["webui"]["port"] = 8001 - -#memcached -default["redborder"]["memcached"]["elasticache"] = false -default["redborder"]["memcached"]["server_list"] = [] -default["redborder"]["memcached"]["options"] = "" -default["redborder"]["memcached"]["port"] = 11211 - -#hadoop -default["redborder"]["hadoop"]["containersMemory"] = 2048 -#samza -default["redborder"]["samza"]["num_containers"] = 1 -default["redborder"]["samza"]["memory_per_container"] = 2560 -#riak +# general +default['redborder']['cdomain'] = 'redborder.cluster' +default['redborder']['organization_uuid'] = nil +default['redborder']['organizations'] = [] +default['redborder']['locations'] = %w(namespace namespace_uuid organization organization_uuid service_provider service_provider_uuid deployment deployment_uuid market market_uuid campus campus_uuid building building_uuid floor floor_uuid) + +# s3 +default['redborder']['uploaded_s3'] = false + +# chef-client +default['chef-client']['interval'] = 300 +default['chef-client']['splay'] = 100 +default['chef-client']['options'] = '' + +# kafka +default['redborder']['kafka']['port'] = 9092 +default['redborder']['kafka']['logdir'] = '/var/log/kafka' +default['redborder']['kafka']['host_index'] = 0 + +# zookeeper +default['redborder']['zookeeper']['zk_hosts'] = '' +default['redborder']['zookeeper']['port'] = 2181 + +# http2k +default['redborder']['http2k']['port'] = 7980 + +# webui +default['redborder']['webui']['port'] = 8001 + +# memcached +default['redborder']['memcached']['elasticache'] = false +default['redborder']['memcached']['server_list'] = [] +default['redborder']['memcached']['options'] = '' +default['redborder']['memcached']['port'] = 11211 + +# hadoop +default['redborder']['hadoop']['containersMemory'] = 2048 +# samza +default['redborder']['samza']['num_containers'] = 1 +default['redborder']['samza']['memory_per_container'] = 2560 +# riak # hard disk -default["redborder"]["manager"]["data_dev"] = {} -default["redborder"]["manager"]["data_dev"]["root"] = "/dev/mapper/VolGroup-lv_root" -default["redborder"]["manager"]["data_dev"]["raw"] = "/dev/mapper/vg_rbdata-lv_raw" -default["redborder"]["manager"]["data_dev"]["aggregate"] = "/dev/mapper/vg_rbdata-lv_aggregated" -default["redborder"]["manager"]["hd_services"] = [ - {"name" => "kafka" , "count" => 5 , "prefered" => "aggregate"}, - {"name" => "zookeeper" , "count" => 1 , "prefered" => "aggregate"}, - {"name" => "riak" , "count" => 50, "prefered" => "raw"}, - {"name" => "druid_historical", "count" => 50, "prefered" => "raw"}, - {"name" => "hadoop_datanode" , "count" => 50, "prefered" => "raw"} +default['redborder']['manager']['data_dev'] = {} +default['redborder']['manager']['data_dev']['root'] = '/dev/mapper/VolGroup-lv_root' +default['redborder']['manager']['data_dev']['raw'] = '/dev/mapper/vg_rbdata-lv_raw' +default['redborder']['manager']['data_dev']['aggregate'] = '/dev/mapper/vg_rbdata-lv_aggregated' +default['redborder']['manager']['hd_services'] = [ + { 'name': 'kafka', 'count': 5, 'prefered': 'aggregate' }, + { 'name': 'zookeeper', 'count': 1, 'prefered': 'aggregate' }, + { 'name': 'riak', 'count': 50, 'prefered': 'raw' }, + { 'name': 'druid_historical', 'count': 50, 'prefered': 'raw' }, + { 'name': 'hadoop_datanode', 'count': 50, 'prefered': 'raw' }, ] -default["redborder"]["manager"]["hd_services_current"] = {} +default['redborder']['manager']['hd_services_current'] = {} # memory -default["redborder"]["memory_services"] = {} -default["redborder"]["memory_services"]["kafka"] = {"count" => 150, "memory" => 0,"max_limit" => 524288} -default["redborder"]["memory_services"]["zookeeper"] = {"count" => 20, "memory" => 0} -default["redborder"]["memory_services"]["chef-client"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["keepalived"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-coordinator"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-overlord"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-historical"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-broker"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-middlemanager"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["druid-realtime"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["http2k"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["chef-server"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["postgresql"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["redborder-postgresql"] = {"count" => 5, "memory" => 0} -default["redborder"]["memory_services"]["mongodb"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["memcached"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["hadoop-nodemanager"] = {"count" => 50, "memory" => 0} -default["redborder"]["memory_services"]["hadoop-resourcemanager"] = {"count" => 10, "memory" => 0} -default["redborder"]["memory_services"]["snmp"] = {"count" => 5, "memory" => 0, "max_limit" => 10000 } -default["redborder"]["memory_services"]["redborder-monitor"] = {"count" => 5, "memory" => 0, "max_limit" => 20000 } -default["redborder"]["memory_services"]["rb-logstatter"] = {"count" => 5, "memory" => 0, "max_limit" => 20000 } -default["redborder"]["memory_services"]["rb-arubacentral"] = {"count" => 5, "memory" => 0, "max_limit" => 20000 } -default["redborder"]["memory_services"]["webui"] = {"count" => 40, "memory" => 0 } -default["redborder"]["memory_services"]["f2k"] = { "count" => 40, "memory" => 0 } -default["redborder"]["memory_services"]["redborder-nmsp"] = {"count" => 10, "memory" => 0 } -default["redborder"]["memory_services"]["n2klocd"] = {"count" => 10, "memory" => 0 } -default["redborder"]["memory_services"]["redborder-cep"] = {"count" => 10, "memory" => 0 } -default["redborder"]["memory_services"]["rb-aioutliers"] = {"count" => 10, "memory" => 0 } +default['redborder']['memory_services'] = {} +default['redborder']['memory_services']['kafka'] = { 'count': 150, 'memory': 0, 'max_limit': 524288 } +default['redborder']['memory_services']['zookeeper'] = { 'count': 20, 'memory': 0 } +default['redborder']['memory_services']['chef-client'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['keepalived'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['druid-coordinator'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['druid-overlord'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['druid-historical'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['druid-broker'] = { 'count' => 10, 'memory' => 0 } +default['redborder']['memory_services']['druid-middlemanager'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['druid-realtime'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['http2k'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['chef-server'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['postgresql'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['redborder-postgresql'] = { 'count': 5, 'memory': 0 } +default['redborder']['memory_services']['mongodb'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['memcached'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['hadoop-nodemanager'] = { 'count': 50, 'memory': 0 } +default['redborder']['memory_services']['hadoop-resourcemanager'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['snmp'] = { 'count': 5, 'memory': 0, 'max_limit': 10000 } +default['redborder']['memory_services']['redborder-monitor'] = { 'count': 5, 'memory': 0, 'max_limit': 20000 } +default['redborder']['memory_services']['rb-logstatter'] = { 'count': 5, 'memory': 0, 'max_limit': 20000 } +default['redborder']['memory_services']['rb-arubacentral'] = { 'count': 5, 'memory': 0, 'max_limit': 20000 } +default['redborder']['memory_services']['webui'] = { 'count': 40, 'memory': 0 } +default['redborder']['memory_services']['f2k'] = { 'count': 40, 'memory': 0 } +default['redborder']['memory_services']['redborder-nmsp'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['n2klocd'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['redborder-cep'] = { 'count': 10, 'memory': 0 } +default['redborder']['memory_services']['rb-aioutliers'] = { 'count': 10, 'memory': 0 } # exclude mem services, setting memory to 0 for each. -default['redborder']['excluded_memory_services'] = %w[chef-client] +default['redborder']['excluded_memory_services'] = %w(chef-client) # default attributes for managers_info, it would be rewriten with the cluster config -default["redborder"]["cluster_info"] = {} -default["redborder"]["cluster_info"][node["hostname"]] = {} -default["redborder"]["cluster_info"][node["hostname"]]["ip"] = node["ipaddress"] +default['redborder']['cluster_info'] = {} +default['redborder']['cluster_info'][node['hostname']] = {} +default['redborder']['cluster_info'][node['hostname']]['ip'] = node['ipaddress'] -default["redborder"]["managers_per_services"] = {} +default['redborder']['managers_per_services'] = {} -default["redborder"]["managers_list"] = ["localhost"] -default["redborder"]["zookeeper_hosts"] = [] +default['redborder']['managers_list'] = ['localhost'] +default['redborder']['zookeeper_hosts'] = [] -default["redborder"]["memory_assigned"] = {} +default['redborder']['memory_assigned'] = {} # geoip has been removed because is not a service -default["redborder"]["services_group"]["full"] = %w[consul chef-server zookeeper memcached rsyslog kafka logstash s3 +default['redborder']['services_group']['full'] = %w(consul chef-server zookeeper memcached rsyslog kafka logstash s3 postgresql redborder-postgresql nginx webui druid-broker druid-historical druid-realtime druid-coordinator f2k redborder-monitor pmacct redborder-dswatcher - redborder-events-counter http2k - ] - -default["redborder"]["services_group"]["custom"] = [] -default["redborder"]["services_group"]["core"] = %w[consul zookeeper druid-coordinator druid-overlord hadoop-resourcemanager] #consul server -default["redborder"]["services_group"]["chef"] = %w[chef-server] -default["redborder"]["services_group"]["kafka"] = %w[kafka] -default["redborder"]["services_group"]["historical"] = %w[druid-historical] -default["redborder"]["services_group"]["middlemanager"] = %w[druid-middlemanager] -default["redborder"]["services_group"]["broker"] = %w[druid-broker] -default["redborder"]["services_group"]["http2k"] = %w[http2k] -default["redborder"]["services_group"]["samza"] = %w[hadoop-nodemanager] -default["redborder"]["services_group"]["webui"] = %w[nginx webui] -default["redborder"]["services_group"]["f2k"] = %w[f2k] -default["redborder"]["services_group"]["s3"] = %w[nginx s3] -default["redborder"]["services_group"]["postgresql"] = %w[postgresql redborder-postgresql] - -default["redborder"]["services"] = {} -default["redborder"]["services"]["chef-client"] = true -default["redborder"]["services"]["chef-server"] = false -default["redborder"]["services"]["consul"] = false -default["redborder"]["services"]["consul-client"] = false -default["redborder"]["services"]["keepalived"] = false -default["redborder"]["services"]["druid-coordinator"] = false -default["redborder"]["services"]["druid-realtime"] = false -default["redborder"]["services"]["druid-historical"] = false -default["redborder"]["services"]["druid-broker"] = false -default["redborder"]["services"]["druid-overlord"] = false -default["redborder"]["services"]["druid-middlemanager"] = false -default["redborder"]["services"]["kafka"] = false -default["redborder"]["services"]["zookeeper"] = false -default["redborder"]["services"]["http2k"] = false -default["redborder"]["services"]["webui"] = false -default["redborder"]["services"]["postgresql"] = false -default["redborder"]["services"]["redborder-postgresql"] = false -default["redborder"]["services"]["nginx"] = false -default["redborder"]["services"]["redborder-cep"] = false -default["redborder"]["services"]["rb-aioutliers"] = false -default["redborder"]["services"]["rb-logstatter"] = true -default["redborder"]["services"]["rb-arubacentral"] = false -default["redborder"]["services"]["memcached"] = true -default["redborder"]["services"]["rb-monitor"] = false -default["redborder"]["services"]["secor"] = false -default["redborder"]["services"]["s3"] = false -default["redborder"]["services"]["mongodb"] = false -default["redborder"]["services"]["hadoop-nodemanager"] = false -default["redborder"]["services"]["hadoop-resourcemanager"] = false -default["redborder"]["services"]["redborder-monitor"] = true -default["redborder"]["services"]["redborder-scanner"] = false -default["redborder"]["services"]["snmp"] = true -default["redborder"]["services"]["f2k"] = false -default["redborder"]["services"]["logstash"] = false -default["redborder"]["services"]["pmacct"] = false -default["redborder"]["services"]["redborder-dswatcher"] = false -default["redborder"]["services"]["redborder-events-counter"] = false -default["redborder"]["services"]["rsyslog"] = true -default["redborder"]["services"]["redborder-nmsp"] = false -default["redborder"]["services"]["redborder-ale"] = false -default["redborder"]["services"]["n2klocd"] = false -default["redborder"]["services"]["radiusd"] = false -default["redborder"]["services"]["postfix"] = true - -default["redborder"]["systemdservices"]["chef-client"] = ["chef-client"] -default["redborder"]["systemdservices"]["chef-server"] = ["opscode-erchef"] -default["redborder"]["systemdservices"]["consul"] = ["consul"] -default["redborder"]["systemdservices"]["consul-client"] = ["consul"] -default["redborder"]["systemdservices"]["druid-realtime"] = ["druid-realtime"] -default["redborder"]["systemdservices"]["druid-coordinator"] = ["druid-coordinator"] -default["redborder"]["systemdservices"]["druid-historical"] = ["druid-historical"] -default["redborder"]["systemdservices"]["druid-broker"] = ["druid-broker"] -default["redborder"]["systemdservices"]["kafka"] = ["kafka"] -default["redborder"]["systemdservices"]["zookeeper"] = ["zookeeper"] -default["redborder"]["systemdservices"]["webui"] = ["webui"] -default["redborder"]["systemdservices"]["postgresql"] = ["postgresql"] -default["redborder"]["systemdservices"]["redborder-postgresql"] = ["redborder-postgresql"] -default["redborder"]["systemdservices"]["nginx"] = ["nginx"] -default["redborder"]["systemdservices"]["redborder-cep"] = ["redborder-cep"] -default["redborder"]["systemdservices"]["rb-aioutliers"] = ["rb-aioutliers"] -default["redborder"]["systemdservices"]["rb-logstatter"] = ["rb-logstatter"] -default["redborder"]["systemdservices"]["rb-arubacentral"] = ["rb-arubacentral"] -default["redborder"]["systemdservices"]["memcached"] = ["memcached"] -default["redborder"]["systemdservices"]["s3"] = ["minio"] -default["redborder"]["systemdservices"]["mongodb"] = ["mongod"] -default["redborder"]["systemdservices"]["redborder-monitor"] = ["redborder-monitor"] -default["redborder"]["systemdservices"]["redborder-scanner"] = ["redborder-scanner"] -default["redborder"]["systemdservices"]["snmp"] = ["snmpd"] -default["redborder"]["systemdservices"]["f2k"] = ["f2k"] -default["redborder"]["systemdservices"]["logstash"] = ["logstash"] -default["redborder"]["systemdservices"]["pmacct"] = ["sfacctd"] -default["redborder"]["systemdservices"]["redborder-dswatcher"] = ["redborder-dswatcher"] -default["redborder"]["systemdservices"]["redborder-events-counter"] = ["redborder-events-counter"] -default["redborder"]["systemdservices"]["http2k"] = ["http2k"] -default["redborder"]["systemdservices"]["rsyslog"] = ["rsyslog"] -default["redborder"]["systemdservices"]["redborder-nmsp"] = ["redborder-nmsp"] -default["redborder"]["systemdservices"]["redborder-ale"] = ["redborder-ale"] -default["redborder"]["systemdservices"]["n2klocd"] = ["n2klocd"] -default["redborder"]["systemdservices"]["radiusd"] = ["radiusd"] -default["redborder"]["systemdservices"]["postfix"] = ["postfix"] + redborder-events-counter http2k) + +default['redborder']['services_group']['custom'] = [] +default['redborder']['services_group']['core'] = %w(consul zookeeper druid-coordinator druid-overlord hadoop-resourcemanager) # consul server +default['redborder']['services_group']['chef'] = %w(chef-server) +default['redborder']['services_group']['kafka'] = %w(kafka) +default['redborder']['services_group']['historical'] = %w(druid-historical) +default['redborder']['services_group']['middlemanager'] = %w(druid-middlemanager) +default['redborder']['services_group']['broker'] = %w(druid-broker) +default['redborder']['services_group']['http2k'] = %w(http2k) +default['redborder']['services_group']['samza'] = %w(hadoop-nodemanager) +default['redborder']['services_group']['webui'] = %w(nginx webui) +default['redborder']['services_group']['f2k'] = %w(f2k) +default['redborder']['services_group']['s3'] = %w(nginx s3) +default['redborder']['services_group']['postgresql'] = %w(postgresql redborder-postgresql) + +default['redborder']['services'] = {} +default['redborder']['services']['chef-client'] = true +default['redborder']['services']['chef-server'] = false +default['redborder']['services']['consul'] = false +default['redborder']['services']['consul-client'] = false +default['redborder']['services']['keepalived'] = false +default['redborder']['services']['druid-coordinator'] = false +default['redborder']['services']['druid-realtime'] = false +default['redborder']['services']['druid-historical'] = false +default['redborder']['services']['druid-broker'] = false +default['redborder']['services']['druid-overlord'] = false +default['redborder']['services']['druid-middlemanager'] = false +default['redborder']['services']['kafka'] = false +default['redborder']['services']['zookeeper'] = false +default['redborder']['services']['http2k'] = false +default['redborder']['services']['webui'] = false +default['redborder']['services']['postgresql'] = false +default['redborder']['services']['redborder-postgresql'] = false +default['redborder']['services']['nginx'] = false +default['redborder']['services']['redborder-cep'] = false +default['redborder']['services']['rb-aioutliers'] = false +default['redborder']['services']['rb-logstatter'] = true +default['redborder']['services']['rb-arubacentral'] = false +default['redborder']['services']['memcached'] = true +default['redborder']['services']['rb-monitor'] = false +default['redborder']['services']['secor'] = false +default['redborder']['services']['s3'] = false +default['redborder']['services']['mongodb'] = false +default['redborder']['services']['hadoop-nodemanager'] = false +default['redborder']['services']['hadoop-resourcemanager'] = false +default['redborder']['services']['redborder-monitor'] = true +default['redborder']['services']['redborder-scanner'] = false +default['redborder']['services']['snmp'] = true +default['redborder']['services']['f2k'] = false +default['redborder']['services']['logstash'] = false +default['redborder']['services']['pmacct'] = false +default['redborder']['services']['redborder-dswatcher'] = false +default['redborder']['services']['redborder-events-counter'] = false +default['redborder']['services']['rsyslog'] = true +default['redborder']['services']['redborder-nmsp'] = false +default['redborder']['services']['redborder-ale'] = false +default['redborder']['services']['n2klocd'] = false +default['redborder']['services']['radiusd'] = false +default['redborder']['services']['postfix'] = true + +default['redborder']['systemdservices']['chef-client'] = ['chef-client'] +default['redborder']['systemdservices']['chef-server'] = ['opscode-erchef'] +default['redborder']['systemdservices']['consul'] = ['consul'] +default['redborder']['systemdservices']['consul-client'] = ['consul'] +default['redborder']['systemdservices']['druid-realtime'] = ['druid-realtime'] +default['redborder']['systemdservices']['druid-coordinator'] = ['druid-coordinator'] +default['redborder']['systemdservices']['druid-historical'] = ['druid-historical'] +default['redborder']['systemdservices']['druid-broker'] = ['druid-broker'] +default['redborder']['systemdservices']['kafka'] = ['kafka'] +default['redborder']['systemdservices']['zookeeper'] = ['zookeeper'] +default['redborder']['systemdservices']['webui'] = ['webui'] +default['redborder']['systemdservices']['postgresql'] = ['postgresql'] +default['redborder']['systemdservices']['redborder-postgresql'] = ['redborder-postgresql'] +default['redborder']['systemdservices']['nginx'] = ['nginx'] +default['redborder']['systemdservices']['redborder-cep'] = ['redborder-cep'] +default['redborder']['systemdservices']['rb-aioutliers'] = ['rb-aioutliers'] +default['redborder']['systemdservices']['rb-logstatter'] = ['rb-logstatter'] +default['redborder']['systemdservices']['rb-arubacentral'] = ['rb-arubacentral'] +default['redborder']['systemdservices']['memcached'] = ['memcached'] +default['redborder']['systemdservices']['s3'] = ['minio'] +default['redborder']['systemdservices']['mongodb'] = ['mongod'] +default['redborder']['systemdservices']['redborder-monitor'] = ['redborder-monitor'] +default['redborder']['systemdservices']['redborder-scanner'] = ['redborder-scanner'] +default['redborder']['systemdservices']['snmp'] = ['snmpd'] +default['redborder']['systemdservices']['f2k'] = ['f2k'] +default['redborder']['systemdservices']['logstash'] = ['logstash'] +default['redborder']['systemdservices']['pmacct'] = ['sfacctd'] +default['redborder']['systemdservices']['redborder-dswatcher'] = ['redborder-dswatcher'] +default['redborder']['systemdservices']['redborder-events-counter'] = ['redborder-events-counter'] +default['redborder']['systemdservices']['http2k'] = ['http2k'] +default['redborder']['systemdservices']['rsyslog'] = ['rsyslog'] +default['redborder']['systemdservices']['redborder-nmsp'] = ['redborder-nmsp'] +default['redborder']['systemdservices']['redborder-ale'] = ['redborder-ale'] +default['redborder']['systemdservices']['n2klocd'] = ['n2klocd'] +default['redborder']['systemdservices']['radiusd'] = ['radiusd'] +default['redborder']['systemdservices']['postfix'] = ['postfix'] # Tier -default["redborder"]["druid"]["historical"]["tier"] = "default" -default["redborder"]["druid"]["historical"]["maxsize"] = -1 +default['redborder']['druid']['historical']['tier'] = 'default' +default['redborder']['druid']['historical']['maxsize'] = -1 # Realtime -default["redborder"]["druid"]["realtime"]["partition_num"] = 0 +default['redborder']['druid']['realtime']['partition_num'] = 0 -default["redborder"]["pending_changes"]=0 +default['redborder']['pending_changes'] = 0 diff --git a/resources/libraries/get_all_flow_sensors_info.rb b/resources/libraries/get_all_flow_sensors_info.rb index 01bfe615..3f2cd85b 100644 --- a/resources/libraries/get_all_flow_sensors_info.rb +++ b/resources/libraries/get_all_flow_sensors_info.rb @@ -1,19 +1,14 @@ -module Rb_manager +module RbManager module Helpers - def get_all_flow_sensors_info() + def get_all_flow_sensors_info sensors_info = {} - sensor_types = ["flow-sensor"] - + sensor_types = ['flow-sensor'] sensor_types.each do |s_type| sensors = search(:node, "role:#{s_type}").sort - sensors_info[s_type] = [] - sensors.each do |s| - sensors_info[s_type] << s - end + sensors.each { |s| sensors_info[s_type] << s } end sensors_info end end end - \ No newline at end of file diff --git a/resources/libraries/get_cluster_info.rb b/resources/libraries/get_cluster_info.rb index abc3f15b..a6afcfc7 100644 --- a/resources/libraries/get_cluster_info.rb +++ b/resources/libraries/get_cluster_info.rb @@ -1,37 +1,34 @@ -module Rb_manager +module RbManager module Helpers - def get_cluster_info() + def get_cluster_info cluster_info = {} - manager_nodes = {} - - manager_nodes = search(:node, "recipes:rb-manager").sort + manager_nodes = search(:node, 'recipes:rb-manager').sort - #The search function above is looking for rb-manager value in "Recipes" key instead run_list, for this reason - #in the first execution the node data is not added to managers hash, so it will be checked now and added - #to managers array - if !cluster_info.key?(node.name) and node.recipe?("rb-manager") and !manager_nodes.include?(node) + # The search function above is looking for rb-manager value in 'Recipes' key instead run_list, for this reason + # in the first execution the node data is not added to managers hash, so it will be checked now and added + # to managers array + if !cluster_info.key?(node.name) && node.recipe?('rb-manager') && !manager_nodes.include?(node) manager_nodes << node end manager_nodes.each do |mnode| name = mnode.name - mnode.normal["rb_time"]=Time.now.to_i if mnode["rb_time"].nil? - rb_time = mnode["rb_time"] + mnode.normal['rb_time'] = Time.now.to_i if mnode['rb_time'].nil? + rb_time = mnode['rb_time'] services = [] # add active services to array - mnode_services = mnode["redborder"]["services"].to_h + mnode_services = mnode['redborder']['services'].to_h mnode_services.each do |service, service_status| services << service if service_status end cluster_info[name] = {} - cluster_info[name]["name"] = name - cluster_info[name]["ip"] = mnode["ipaddress"] - cluster_info[name]["rb_time"] = rb_time - cluster_info[name]["services"] = services + cluster_info[name]['name'] = name + cluster_info[name]['ip'] = mnode['ipaddress'] + cluster_info[name]['rb_time'] = rb_time + cluster_info[name]['services'] = services end - cluster_info = cluster_info.sort{|a,b| (a[1]["rb_time"]||999999999999999999999) <=> (b[1]["rb_time"]||999999999999999999999)}.to_h - return cluster_info + cluster_info = cluster_info.sort { |a, b| (a[1]['rb_time'] || 999999999999999999999) <=> (b[1]['rb_time'] || 999999999999999999999) }.to_h end end end diff --git a/resources/libraries/get_elasticache_nodes.rb b/resources/libraries/get_elasticache_nodes.rb index c2c5821a..095f393f 100755 --- a/resources/libraries/get_elasticache_nodes.rb +++ b/resources/libraries/get_elasticache_nodes.rb @@ -1,6 +1,5 @@ -module Rb_manager +module RbManager module Helpers - require 'timeout' require 'socket' @@ -8,20 +7,17 @@ def getElasticacheNodes(config_endpoint, port) begin socket = TCPSocket.new config_endpoint, port response = [] - status = Timeout::timeout(2) { + Timeout.timeout(2) do finish = true - socket.puts("config get cluster") + socket.puts('config get cluster') while finish response.push(socket.gets.chomp) - finish = false if "#{response.last}" == "END" + finish = false if response.last == 'END' end - } + end socket.close - nodes = response.at(2).split(" ").map { - |server| - server.split("|").at(0) - } - return nodes + + response.at(2).split(' ').map { |server| server.split('|').at(0) } rescue => e Chef::Log.error(e.message) end diff --git a/resources/libraries/get_managers_all.rb b/resources/libraries/get_managers_all.rb index 1df2fbe4..ea4e2949 100644 --- a/resources/libraries/get_managers_all.rb +++ b/resources/libraries/get_managers_all.rb @@ -1,4 +1,4 @@ -module Rb_manager +module RbManager module Helpers def get_managers_all managers = [] @@ -15,13 +15,14 @@ def get_managers_all roles = [] end end - unless roles.nil? - if roles.include?("manager") - managers << m - end - end + next unless roles.nil? + + next unless roles.include?('manager') + + managers << m end + managers end end -end \ No newline at end of file +end diff --git a/resources/libraries/get_namespaces.rb b/resources/libraries/get_namespaces.rb index 9cca6ec7..809c5f84 100644 --- a/resources/libraries/get_namespaces.rb +++ b/resources/libraries/get_namespaces.rb @@ -1,14 +1,15 @@ -module Rb_manager +module RbManager module Helpers def get_namespaces namespaces = [] - Chef::Role.list.keys.each do |rol| + Chef::Role.list.each_key do |rol| ro = Chef::Role.load rol - if ro and ro.override_attributes["redborder"] and ro.override_attributes["redborder"]["namespace"] and ro.override_attributes["redborder"]["namespace_uuid"] and !ro.override_attributes["redborder"]["namespace_uuid"].empty? - namespaces.push(ro.override_attributes["redborder"]["namespace_uuid"]) - end + next unless ro && ro.override_attributes['redborder'] && ro.override_attributes['redborder']['namespace'] && ro.override_attributes['redborder']['namespace_uuid'] && !ro.override_attributes['redborder']['namespace_uuid'].empty? + + namespaces.push(ro.override_attributes['redborder']['namespace_uuid']) end + namespaces.uniq end end -end \ No newline at end of file +end diff --git a/resources/libraries/get_orgs.rb b/resources/libraries/get_orgs.rb index 213aa288..22599c95 100644 --- a/resources/libraries/get_orgs.rb +++ b/resources/libraries/get_orgs.rb @@ -1,19 +1,16 @@ -module Rb_manager +module RbManager module Helpers - def get_orgs() - + def get_orgs organizations = [] - Chef::Role.list.keys.each do |m_key| + Chef::Role.list.each_key do |m_key| m = Chef::Role.load m_key - if !m.override_attributes["redborder"].nil? and !m.override_attributes["redborder"]["organization_uuid"].nil? and m.override_attributes["redborder"]["sensor_uuid"] == m.override_attributes["redborder"]["organization_uuid"] - organizations << m - end + next unless m.override_attributes['redborder'] && m.override_attributes['redborder']['organization_uuid'] && m.override_attributes['redborder']['sensor_uuid'] == m.override_attributes['redborder']['organization_uuid'] + + organizations << m end organizations - end end end - diff --git a/resources/libraries/get_pipelines.rb b/resources/libraries/get_pipelines.rb index f9fbeaa3..770eebed 100644 --- a/resources/libraries/get_pipelines.rb +++ b/resources/libraries/get_pipelines.rb @@ -1,27 +1,26 @@ -module Rb_manager +module RbManager module Helpers - - def get_pipelines() + def get_pipelines logstash_pipelines = [] - sensors = node.run_state["sensors_info_all"] + sensors = node.run_state['sensors_info_all'] namespaces = get_namespaces() main_logstash = determine_main_logstash_node() - if manager_services["logstash"] - logstash_pipelines.push("rbwindow-pipeline") if main_logstash == node.name - logstash_pipelines.push("apstate-pipeline") - logstash_pipelines.push("scanner-pipeline") unless sensors["scanner-sensor"].empty? - logstash_pipelines.push("nmsp-pipeline") if main_logstash == node.name and !sensors["flow-sensor"].empty? - logstash_pipelines.push("radius-pipeline") if main_logstash == node.name - logstash_pipelines.push("vault-pipeline") unless sensors["vault-sensor"].empty? - logstash_pipelines.push("netflow-pipeline") unless sensors["flow-sensor"].empty? - logstash_pipelines.push("sflow-pipeline") unless sensors["flow-sensor"].empty? - logstash_pipelines.push("meraki-pipeline") unless sensors["meraki-sensor"].empty? - logstash_pipelines.push("monitor-pipeline") unless namespaces.empty? - logstash_pipelines.push("location-pipeline") unless sensors["ale-sensor"].empty? or sensors["mse-sensor"].empty? or sensors["flow-sensor"].empty? or sensors["arubacentral-sensor"].empty? - logstash_pipelines.push("mobility-pipeline") - logstash_pipelines.push("redfish-pipeline") unless sensors["device-sensor"].empty? - logstash_pipelines.push("bulkstats-pipeline") unless sensors["device-sensor"].empty? + if manager_services['logstash'] + logstash_pipelines.push('rbwindow-pipeline') if main_logstash == node.name + logstash_pipelines.push('apstate-pipeline') + logstash_pipelines.push('scanner-pipeline') unless sensors['scanner-sensor'].empty? + logstash_pipelines.push('nmsp-pipeline') if main_logstash == node.name && !sensors['flow-sensor'].empty? + logstash_pipelines.push('radius-pipeline') if main_logstash == node.name + logstash_pipelines.push('vault-pipeline') unless sensors['vault-sensor'].empty? + logstash_pipelines.push('netflow-pipeline') unless sensors['flow-sensor'].empty? + logstash_pipelines.push('sflow-pipeline') unless sensors['flow-sensor'].empty? + logstash_pipelines.push('meraki-pipeline') unless sensors['meraki-sensor'].empty? + logstash_pipelines.push('monitor-pipeline') unless namespaces.empty? + logstash_pipelines.push('location-pipeline') unless sensors['ale-sensor'].empty? || sensors['mse-sensor'].empty? || sensors['flow-sensor'].empty? || sensors['arubacentral-sensor'].empty? + logstash_pipelines.push('mobility-pipeline') + logstash_pipelines.push('redfish-pipeline') unless sensors['device-sensor'].empty? + logstash_pipelines.push('bulkstats-pipeline') unless sensors['device-sensor'].empty? end logstash_pipelines end @@ -29,9 +28,9 @@ def get_pipelines() # The main logstash is a node where both memcached and logstash are running or the first logstash node (order by name). # This main logasths node is gonna run the pipelines rbwindow, location, mobility, nmsp, meraki and radius # Those pipelines should only run in one node (the main_logstash node) for now.. - def determine_main_logstash_node() - memcached_nodes = managers_per_service["memcached"].sort.uniq - logstash_nodes = managers_per_service["logstash"].sort.uniq + def determine_main_logstash_node + memcached_nodes = managers_per_service['memcached'].sort.uniq + logstash_nodes = managers_per_service['logstash'].sort.uniq main_logstash_nodes = memcached_nodes & logstash_nodes main_logstash_nodes.first || logstash_nodes.first end diff --git a/resources/libraries/get_sensors_all_info.rb b/resources/libraries/get_sensors_all_info.rb index a4e1bede..9a4ea65e 100644 --- a/resources/libraries/get_sensors_all_info.rb +++ b/resources/libraries/get_sensors_all_info.rb @@ -1,17 +1,16 @@ -module Rb_manager +module RbManager module Helpers - def get_sensors_all_info() + def get_sensors_all_info sensors_info = {} - sensor_types = ["ips-sensor","ipsv2-sensor","ipscp-sensor","ipsg-sensor","vault-sensor","flow-sensor","arubacentral-sensor","mse-sensor","meraki-sensor","cisco-cloudproxy","proxy-sensor","scanner-sensor","mse-sensor","meraki-sensor","ale-sensor","cep-sensor","device-sensor"] + sensor_types = %w(ips-sensor ipsv2-sensor ipscp-sensor ipsg-sensor vault-sensor flow-sensor arubacentral-sensor mse-sensor meraki-sensor cisco-cloudproxy proxy-sensor scanner-sensor mse-sensor meraki-sensor ale-sensor cep-sensor device-sensor) sensor_types.each do |s_type| - sensors = search(:node, "role:#{s_type} AND -redborder_parent_id:*?").sort #get sensor where parent_id is nil + sensors = search(:node, "role:#{s_type} AND -redborder_parent_id:*?").sort # get sensor where parent_id is nil sensors_info[s_type] = [] - sensors.each do |s| - sensors_info[s_type] << s - end + sensors.each { |s| sensors_info[s_type] << s } end + sensors_info end end diff --git a/resources/libraries/get_sensors_info.rb b/resources/libraries/get_sensors_info.rb index 30fe0d7a..dac826ea 100644 --- a/resources/libraries/get_sensors_info.rb +++ b/resources/libraries/get_sensors_info.rb @@ -1,34 +1,37 @@ -module Rb_manager +module RbManager module Helpers - def get_sensors_info() + def get_sensors_info sensors_info = {} sensor_types = %w(vault-sensor flow-sensor mse-sensor scanner-sensor meraki-sensor ale-sensor device-sensor cisco-cloudproxy proxy-sensor ips-sensor ipsv2-sensor ipscp-sensor ipsg-sensor) - locations = node["redborder"]["locations"] + locations = node['redborder']['locations'] sensor_types.each do |s_type| - sensors = search(:node, "role:#{s_type} AND -redborder_parent_id:*?").sort #get sensor where parent_id is nil + # get sensor where parent_id is nil + sensors = search(:node, "role:#{s_type} AND -redborder_parent_id:*?").sort sensors_info[s_type] = {} sensors.each do |s| info = {} - info["name"] = s.name - info["ip"] = s["ipaddress"] - info["sensor_uuid"] = s["redborder"]["sensor_uuid"] if !s["redborder"]["sensor_uuid"].nil? - info["organization_uuid"] = s["redborder"]["organization_uuid"] if !s["redborder"]["organization_uuid"].nil? - info["megabytes_limit"] = s["redborder"]["megabytes_limit"] if !s["redborder"]["megabytes_limit"].nil? - info["index_partitions"] = s["redborder"]["index_partitions"] if !s["redborder"]["index_partitions"].nil? - info["index_replicas"] = s["redborder"]["index_replicas"] if !s["redborder"]["index_replicas"].nil? - info["sensors_mapping"] = s["redborder"]["sensors_mapping"] if !s["redborder"]["sensors_mapping"].nil? - info["locations"] = {} + info['name'] = s.name + info['ip'] = s['ipaddress'] + info['sensor_uuid'] = s['redborder']['sensor_uuid'] if s['redborder']['sensor_uuid'] + info['organization_uuid'] = s['redborder']['organization_uuid'] if s['redborder']['organization_uuid'] + info['megabytes_limit'] = s['redborder']['megabytes_limit'] if s['redborder']['megabytes_limit'] + info['index_partitions'] = s['redborder']['index_partitions'] if s['redborder']['index_partitions'] + info['index_replicas'] = s['redborder']['index_replicas'] if s['redborder']['index_replicas'] + info['sensors_mapping'] = s['redborder']['sensors_mapping'] if s['redborder']['sensors_mapping'] + info['locations'] = {} + locations.each do |loc| - if !s["redborder"][loc].nil? - info["locations"][loc] = s["redborder"][loc] - end + next unless s['redborder'][loc] + + info['locations'][loc] = s['redborder'][loc] end sensors_info[s_type][s.name] = info end end - return sensors_info + + sensors_info end end end diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index ff98544b..8d15b555 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -1,73 +1,74 @@ -module Rb_manager +module RbManager module Helpers - def harddisk_services() + def harddisk_services hd_services = {} - root_dev=node["redborder"]["manager"]["data_dev"]["root"] - raw_dev=node["redborder"]["manager"]["data_dev"]["raw"] - aggregate_dev=node["redborder"]["manager"]["data_dev"]["aggregate"] - - root_dev=node["filesystem"].keys.first if node["filesystem"][root_dev].nil? - if node["filesystem"][raw_dev].nil? and node["filesystem"][aggregate_dev].nil? - raw_dev=root_dev - aggregate_dev=root_dev - elsif node["filesystem"][raw_dev].nil? - raw_dev=aggregate_dev - elsif node["filesystem"][aggregate_dev].nil? - aggregate_dev=raw_dev + root_dev = node['redborder']['manager']['data_dev']['root'] + raw_dev = node['redborder']['manager']['data_dev']['raw'] + aggregate_dev = node['redborder']['manager']['data_dev']['aggregate'] + + root_dev = node['filesystem'].keys.first if node['filesystem'][root_dev].nil? + if node['filesystem'][raw_dev].nil? && node['filesystem'][aggregate_dev].nil? + raw_dev = root_dev + aggregate_dev = root_dev + elsif node['filesystem'][raw_dev].nil? + raw_dev = aggregate_dev + elsif node['filesystem'][aggregate_dev].nil? + aggregate_dev = raw_dev end - + hd_services_dev = {} - hd_services_dev["root"] = root_dev - hd_services_dev["raw"] = raw_dev - hd_services_dev["aggregate"] = aggregate_dev - + hd_services_dev['root'] = root_dev + hd_services_dev['raw'] = raw_dev + hd_services_dev['aggregate'] = aggregate_dev + hd_services_size = {} - hd_services_size[root_dev] =0 - hd_services_size[raw_dev] =0 - hd_services_size[aggregate_dev]=0 - + hd_services_size[root_dev] = 0 + hd_services_size[raw_dev] = 0 + hd_services_size[aggregate_dev] = 0 + hd_services_size_total = {} - hd_services_size_total[root_dev] = 0 - hd_services_size_total[raw_dev] = 0 + hd_services_size_total[root_dev] = 0 + hd_services_size_total[raw_dev] = 0 hd_services_size_total[aggregate_dev] = 0 - + maxsize = {} hd_services_dev.each do |type, device| - if node["filesystem"][device].nil? - maxsize[type] = 300000000000 - else - if node["filesystem"][device]["kb_size"].nil? and !node["filesystem"][hd_services_dev["root"]]["kb_size"].nil? - maxsize[type] = node["filesystem"][hd_services_dev["root"]]["kb_size"].to_i*1024 - else - maxsize[type] = node["filesystem"][device]["kb_size"].to_i*1024 - end - end - maxsize[type] = maxsize[type] - 7*1024*1024*1024 if device==hd_services_dev["root"] - maxsize[type] = 0 if maxsize[type]<0 + maxsize[type] = if node['filesystem'][device].nil? + 300_000_000_000 + else + filesystem_data = node['filesystem'][device] + root_device = hd_services_dev['root'] + size_kb = filesystem_data['kb_size'] || node['filesystem'][root_device]['kb_size'] + size_bytes = size_kb.to_i * 1024 + size_bytes -= 7 * 1024 * 1024 * 1024 if device == root_device + [size_bytes, 0].max + end end - - node["redborder"]["manager"]["hd_services"].each do |s| - if node["redborder"]["services"][s[:name]] - hd_services_size[hd_services_dev[s[:prefered]]] = hd_services_size[hd_services_dev[s[:prefered]]] + s[:count] + node['redborder']['manager']['hd_services'].each do |s| + if node['redborder']['services'][s[:name]] + hd_services_size[hd_services_dev[s[:prefered]]] += s[:count] end - hd_services_size_total[hd_services_dev[s[:prefered]]] = hd_services_size_total[hd_services_dev[s[:prefered]]] + s[:count] + hd_services_size_total[hd_services_dev[s[:prefered]]] += s[:count] end - - node["redborder"]["manager"]["hd_services"].each do |s| - if hd_services_size[hd_services_dev[s[:prefered]]]>0 - size = hd_services_size[hd_services_dev[s[:prefered]]] - else - if hd_services_size_total[hd_services_dev[s[:prefered]]]>0 - size = hd_services_size_total[hd_services_dev[s[:prefered]]] - else - size =1 - end - end - hd_services[s[:name]] = ((s[:count].to_i * maxsize[s[:prefered]].to_i * 0.90)/size) / 1024 * 1024 - end - return hd_services + node['redborder']['manager']['hd_services'].each do |service| + preferred_device = hd_services_dev[service[:prefered]] + preferred_size = hd_services_size[preferred_device] + total_size = hd_services_size_total[preferred_device] + + size = if preferred_size > 0 + preferred_size + else + total_size > 0 ? total_size : 1 + end + + service_count = service[:count].to_i + max_preferred_size = maxsize[service[:prefered]].to_i + hd_services[service[:name]] = ((service_count * max_preferred_size * 0.90) / size.to_f) / (1024 * 1024) + end + + hd_services end end end diff --git a/resources/libraries/in_hash.rb b/resources/libraries/in_hash.rb index 1babd76b..44a512ba 100644 --- a/resources/libraries/in_hash.rb +++ b/resources/libraries/in_hash.rb @@ -1,15 +1,13 @@ class Chef class Recipe - - def in_hash?(hash, *keys) + def in_hash?(hash, *keys) if hash[keys.first] && keys.size == 1 - return true + true elsif hash[keys.first] && hash[keys.first].is_a?(Hash) - in_hash? hash[keys.first], *keys[1..keys.size-1] + in_hash? hash[keys.first], *keys[1..keys.size - 1] else - return false + false end end - end end diff --git a/resources/libraries/manager_services.rb b/resources/libraries/manager_services.rb index fd32e0aa..ac29c639 100644 --- a/resources/libraries/manager_services.rb +++ b/resources/libraries/manager_services.rb @@ -1,13 +1,25 @@ class Chef class Recipe - def manager_services() - manager_services = {} - node["redborder"]["services"].each { |k,v| manager_services[k] = v if (v==true or v==false) } if !node["redborder"]["services"].nil? - + def manager_services + manager_services = {} + if node['redborder']['services'] + node['redborder']['services'].each do |k, v| + if v == true || v == false + manager_services[k] = v + end + end + end + # changing default values in case of the user has modify them - node["redborder"]["services"]["overwrite"].each { |k,v| manager_services[k] = v if (v==true or v==false) } if !node["redborder"]["services"]["overwrite"].nil? - - return manager_services + if node['redborder']['services']['overwrite'] + node['redborder']['services']['overwrite'].each do |k, v| + if v == true || v == false + manager_services[k] = v + end + end + end + + manager_services end end -end \ No newline at end of file +end diff --git a/resources/libraries/managers_per_service.rb b/resources/libraries/managers_per_service.rb index 5dac8128..e52865b1 100644 --- a/resources/libraries/managers_per_service.rb +++ b/resources/libraries/managers_per_service.rb @@ -1,19 +1,20 @@ class Chef class Recipe - def managers_per_service() - services = node["redborder"]["services"] - cluster_info = node["redborder"]["cluster_info"] + def managers_per_service + services = node['redborder']['services'] + cluster_info = node['redborder']['cluster_info'] cluster_services = {} - services.each do |serv, status| + services.each do |serv, _status| cluster_services[serv] = [] cluster_info.each do |manager, info| - if !info["services"].nil? and info["services"].include?(serv) - cluster_services[serv] << manager + if info['services'] && info['services'].include?(serv) + cluster_services[serv] << manager end end end - return cluster_services + + cluster_services end end end diff --git a/resources/libraries/memory_services.rb b/resources/libraries/memory_services.rb index 03f7eb66..90b0e7ea 100644 --- a/resources/libraries/memory_services.rb +++ b/resources/libraries/memory_services.rb @@ -1,53 +1,50 @@ -module Rb_manager +module RbManager module Helpers - def memory_services(sysmem_total, excluded_services=[]) + def memory_services(sysmem_total, excluded_services = []) memory_serv = {} memory_services_size = 0 memory_services_size_total = 0 memlimit_found = false sysmem_total_limitsless = nil - - node["redborder"]["memory_services"].each do |name,mem_s| - if node["redborder"]["services"][name] and !excluded_services.include?(name) - if !node["redborder"]["excluded_memory_services"].include?(name) - memory_services_size = memory_services_size + mem_s["count"] + + node['redborder']['memory_services'].each do |name, mem_s| + if node['redborder']['services'][name] && !excluded_services.include?(name) + unless node['redborder']['excluded_memory_services'].include?(name) + memory_services_size += mem_s['count'] end end - memory_services_size_total = memory_services_size_total + mem_s["count"] + memory_services_size_total += mem_s['count'] end + if memory_services_size <= 0 - if memory_services_size_total > 0 - memory_services_size = memory_services_size_total - else - memory_services_size = 1 - end + memory_services_size = memory_services_size_total > 0 ? memory_services_size_total : 1 end - node["redborder"]["memory_services"].each do |name,mem_s| - - if node["redborder"]["services"][name] and !excluded_services.include?(name) - if !node["redborder"]["excluded_memory_services"].include?(name) - # service count memory assigned * system memory / assigned services memory size - memory_serv[name] = (mem_s["count"] * sysmem_total / memory_services_size).round - #if the service has a limit of memory, we have to recalculate all using recursivity - if !mem_s["max_limit"].nil? and memory_serv[name] > mem_s["max_limit"] - memlimit_found = true - excluded_services << name - #assigning the limit of memory for this service - node.default["redborder"]["memory_services"][name]["memory"] = mem_s["max_limit"] - #now we have to take off the memory excluded from the total to recalculate memory wihout excluded services by limit - sysmem_total_limitsless = sysmem_total - mem_s["max_limit"] - end - end - end + node['redborder']['memory_services'].each do |name, mem_s| + next unless node['redborder']['services'][name] && !excluded_services.include?(name) + + next unless !node['redborder']['excluded_memory_services'].include?(name) + + # service count memory assigned * system memory / assigned services memory size + memory_serv[name] = (mem_s['count'] * sysmem_total / memory_services_size).round + + # if the service has a limit of memory, we have to recalculate all using recursivity + next unless mem_s['max_limit'] && memory_serv[name] > mem_s['max_limit'] + + memlimit_found = true + excluded_services << name + # assigning the limit of memory for this service + node.default['redborder']['memory_services'][name]['memory'] = mem_s['max_limit'] + # now we have to take off the memory excluded from the total to recalculate memory wihout excluded services by limit + sysmem_total_limitsless = sysmem_total - mem_s['max_limit'] end if memlimit_found - #Function that call itself with services excluded for recalculate memory + # function that call itself with services excluded for recalculate memory memory_serv = memory_services(sysmem_total_limitsless, excluded_services) else memory_serv.each do |name, memory| - node.default["redborder"]["memory_services"][name]["memory"] = memory + node.default['redborder']['memory_services'][name]['memory'] = memory end end end diff --git a/resources/libraries/node_services.rb b/resources/libraries/node_services.rb index e3e2bc0e..bfe3145b 100644 --- a/resources/libraries/node_services.rb +++ b/resources/libraries/node_services.rb @@ -1,19 +1,19 @@ -module Rb_manager +module RbManager module Helpers def node_services(rbnode) - if !rbnode.nil? and !rbnode.empty? - rbnodes_arr = [] - rbnodes_arr = search(:node, "name:#{rbnode}") - if rbnodes_arr.length == 1 - rbnode_attributes = rbnodes_arr.first - services = [] - node["redborder"]["services"].each do |service, service_status| - services << service if service_status - end - return services - else - raise "ERROR: There are two nodes with the same name, Im not able to get the #{manager} services." + return unless rbnode && !rbnode.empty? + + rbnodes_arr = search(:node, "name:#{rbnode}") + if rbnodes_arr.length == 1 + # rbnode_attributes = rbnodes_arr.first + services = [] + node['redborder']['services'].each do |service, service_status| + services << service if service_status end + + services + else + raise "ERROR: There are two nodes with the same name, Im not able to get the #{manager} services." end end end diff --git a/resources/libraries/util.rb b/resources/libraries/util.rb index 913c15a4..56c41307 100644 --- a/resources/libraries/util.rb +++ b/resources/libraries/util.rb @@ -1,10 +1,9 @@ -module Rb_manager +module RbManager module Helpers - def joinHostArray2port(hosts, port) - hosts.map { |host| - host << ":" << port.to_s - } - return hosts + def joinHostArray2port(hosts, port) + hosts.map { |host| host << ':' << port.to_s } + + hosts end end -end \ No newline at end of file +end diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 579301ee..b2c1119a 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -1,460 +1,609 @@ -# -# Cookbook Name:: manager +# Cookbook:: manager # Recipe:: configure -# -# Copyright 2024, redborder -# -# AFFERO GENERAL PUBLIC LICENSE V3 -# +# Copyright:: 2024, redborder +# License:: Affero General Public License, Version 3 # Services configuration # manager services manager_services = manager_services() -rb_common_config "Configure common" do +rb_common_config 'Configure common' do action :configure end -rb_selinux_config "Configure Selinux" do - if shell_out("getenforce").stdout.chomp == "Disabled" +rb_selinux_config 'Configure Selinux' do + if shell_out('getenforce').stdout.chomp == 'Disabled' action :remove else action :add end end -consul_config "Configure Consul Server" do - confdir node["consul"]["confdir"] - datadir node["consul"]["datadir"] - ipaddress node["ipaddress_sync"] - cdomain node["redborder"]["cdomain"] - dns_local_ip node["consul"]["dns_local_ip"] - (manager_services["consul"] ? (is_server true) : (is_server false)) - action ((manager_services["consul"] or manager_services["consul-client"]) ? :add : :remove) +consul_config 'Configure Consul Server' do + confdir node['consul']['confdir'] + datadir node['consul']['datadir'] + ipaddress node['ipaddress_sync'] + cdomain node['redborder']['cdomain'] + dns_local_ip node['consul']['dns_local_ip'] + (manager_services['consul'] ? (is_server true) : (is_server false)) + if manager_services['consul'] || manager_services['consul-client'] + action :add + else + action :remove + end end -if manager_services["chef-server"] - chef_server_config "Configure chef services" do - memory node["redborder"]["memory_services"]["chef-server"]["memory"] - postgresql false - postgresql_memory node["redborder"]["memory_services"]["postgresql"]["memory"] - chef_active manager_services["chef-server"] - ipaddress node["ipaddress_sync"] +chef_server_config 'Configure chef services' do + memory node['redborder']['memory_services']['chef-server']['memory'] + postgresql false + postgresql_memory node['redborder']['memory_services']['postgresql']['memory'] + chef_active manager_services['chef-server'] + ipaddress node['ipaddress_sync'] + if manager_services['chef-server'] action [:add, :register] - end -else - chef_server_config "Remove chef service" do + else action [:remove, :deregister] end end -zookeeper_config "Configure Zookeeper" do - port node["zookeeper"]["port"] - memory node["redborder"]["memory_services"]["zookeeper"]["memory"] - hosts node["redborder"]["managers_per_services"]["zookeeper"] - ipaddress node["ipaddress_sync"] - action (manager_services["zookeeper"] ? [:add, :register] : [:remove, :deregister]) +zookeeper_config 'Configure Zookeeper' do + port node['zookeeper']['port'] + memory node['redborder']['memory_services']['zookeeper']['memory'] + hosts node['redborder']['managers_per_services']['zookeeper'] + ipaddress node['ipaddress_sync'] + if manager_services['zookeeper'] + action [:add, :register] + else + action [:remove, :deregister] + end end -kafka_config "Configure Kafka" do - memory node["redborder"]["memory_services"]["kafka"]["memory"] - maxsize node["redborder"]["manager"]["hd_services_current"]["kafka"] - managers_list node["redborder"]["managers_per_services"]["kafka"] - zk_hosts node["redborder"]["zookeeper"]["zk_hosts"] - host_index node["redborder"]["kafka"]["host_index"] - ipaddress node["ipaddress_sync"] - action (manager_services["kafka"] ? [:add, :register] : [:remove, :deregister]) +kafka_config 'Configure Kafka' do + memory node['redborder']['memory_services']['kafka']['memory'] + maxsize node['redborder']['manager']['hd_services_current']['kafka'] + managers_list node['redborder']['managers_per_services']['kafka'] + zk_hosts node['redborder']['zookeeper']['zk_hosts'] + host_index node['redborder']['kafka']['host_index'] + ipaddress node['ipaddress_sync'] + if manager_services['kafka'] + action [:add, :register] + else + action [:remove, :deregister] + end end -if manager_services["druid-coordinator"] or - manager_services["druid-overlord"] or - manager_services["druid-broker"] or - manager_services["druid-middlemanager"] or - manager_services["druid-historical"] or - manager_services["druid-realtime"] - - ["druid-broker", "druid-coordinator", "druid-historical", - "druid-middlemanager", "druid-overlord"].each do |druid_service| +if manager_services['druid-coordinator'] || manager_services['druid-overlord'] || manager_services['druid-broker'] || manager_services['druid-middlemanager'] || manager_services['druid-historical'] || manager_services['druid-realtime'] + %w(druid-broker druid-coordinator druid-historical + druid-middlemanager druid-overlord).each do |druid_service| service druid_service do - supports :status => true, :start => true, :restart => true, :reload => true + supports status: true, start: true, restart: true, reload: true action :nothing end end - druid_common "Configure druid common resources" do - name node["hostname"] - zookeeper_hosts node["redborder"]["zookeeper"]["zk_hosts"] - memcached_hosts node["redborder"]["memcached"]["hosts"] - s3_service "s3.service" - s3_port node["minio"]["port"] - cdomain node["redborder"]["cdomain"] + druid_common 'Configure druid common resources' do + name node['hostname'] + zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] + memcached_hosts node['redborder']['memcached']['hosts'] + s3_service 's3.service' + s3_port node['minio']['port'] + cdomain node['redborder']['cdomain'] action :add - notifies :restart, 'service[druid-broker]', :delayed if manager_services["druid-broker"] - notifies :restart, 'service[druid-coordinator]', :delayed if manager_services["druid-coordinator]"] - notifies :restart, 'service[druid-historical]', :delayed if manager_services["druid-historical"] - notifies :restart, 'service[druid-middlemanager]', :delayed if manager_services["druid-middlemanager"] - notifies :restart, 'service[druid-overlord]', :delayed if manager_services["druid-overlord"] + notifies :restart, 'service[druid-broker]', :delayed if manager_services['druid-broker'] + notifies :restart, 'service[druid-coordinator]', :delayed if manager_services['druid-coordinator]'] + notifies :restart, 'service[druid-historical]', :delayed if manager_services['druid-historical'] + notifies :restart, 'service[druid-middlemanager]', :delayed if manager_services['druid-middlemanager'] + notifies :restart, 'service[druid-overlord]', :delayed if manager_services['druid-overlord'] end else - druid_common "Delete druid common resources" do + druid_common 'Delete druid common resources' do action :remove end end -druid_coordinator "Configure Druid Coordinator" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - memory_kb node["redborder"]["memory_services"]["druid-coordinator"]["memory"] - action (manager_services["druid-coordinator"] ? [:add, :register] : [:remove, :deregister]) +druid_coordinator 'Configure Druid Coordinator' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-coordinator']['memory'] + if manager_services['druid-coordinator'] + action [:add, :register] + else + action [:remove, :deregister] + end end -druid_overlord "Configure Druid Overlord" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - memory_kb node["redborder"]["memory_services"]["druid-overlord"]["memory"] - action (manager_services["druid-overlord"] ? [:add, :register] : [:remove, :deregister]) +druid_overlord 'Configure Druid Overlord' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-overlord']['memory'] + if manager_services['druid-overlord'] + action [:add, :register] + else + action [:remove, :deregister] + end end -druid_broker "Configure Druid Broker" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - memory_kb node["redborder"]["memory_services"]["druid-broker"]["memory"] - action (manager_services["druid-broker"] ? [:add, :register] : [:remove, :deregister]) +druid_broker 'Configure Druid Broker' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-broker']['memory'] + if manager_services['druid-broker'] + action [:add, :register] + else + action [:remove, :deregister] + end end -druid_middlemanager "Configure Druid MiddleManager" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - memory_kb node["redborder"]["memory_services"]["druid-middlemanager"]["memory"] - action (manager_services["druid-middlemanager"] ? [:add, :register] : [:remove, :deregister]) +druid_middlemanager 'Configure Druid MiddleManager' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-middlemanager']['memory'] + if manager_services['druid-middlemanager'] + action [:add, :register] + else + action [:remove, :deregister] + end end -druid_historical "Configure Druid Historical" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - memory_kb node["redborder"]["memory_services"]["druid-historical"]["memory"] - action (manager_services["druid-historical"] ? [:add, :register] : [:remove, :deregister]) +druid_historical 'Configure Druid Historical' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-historical']['memory'] + if manager_services['druid-historical'] + action [:add, :register] + else + action [:remove, :deregister] + end end -druid_realtime "Configure Druid Realtime" do - name node["hostname"] - ipaddress node["ipaddress_sync"] - zookeeper_hosts node["redborder"]["zookeeper"]["zk_hosts"] - partition_num node["redborder"]["druid"]["realtime"]["partition_num"] - memory_kb node["redborder"]["memory_services"]["druid-realtime"]["memory"] - action (manager_services["druid-realtime"] ? [:add, :register] : [:remove, :deregister]) +druid_realtime 'Configure Druid Realtime' do + name node['hostname'] + ipaddress node['ipaddress_sync'] + zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] + partition_num node['redborder']['druid']['realtime']['partition_num'] + memory_kb node['redborder']['memory_services']['druid-realtime']['memory'] + if manager_services['druid-realtime'] + action [:add, :register] + else + action [:remove, :deregister] + end end -memcached_config "Configure Memcached" do - memory node["redborder"]["memory_services"]["memcached"]["memory"] - ipaddress node["ipaddress_sync"] - action (manager_services["memcached"] ? [:add, :register] : [:remove, :deregister]) +memcached_config 'Configure Memcached' do + memory node['redborder']['memory_services']['memcached']['memory'] + ipaddress node['ipaddress_sync'] + if manager_services['memcached'] + action [:add, :register] + else + action [:remove, :deregister] + end end -mongodb_config "Configure Mongodb" do - action (manager_services["mongodb"] ? [:add, :register] : [:remove, :deregister]) +mongodb_config 'Configure Mongodb' do + if manager_services['mongodb'] + action [:add, :register] + else + action [:remove, :deregister] + end end -if manager_services["hadoop-nodemanager"] or - manager_services["hadoop-resourcemanager"] - - hadoop_common "Configure hadoop common resources" do - name node["hostname"] - zookeeper_hosts node["redborder"]["zookeeper"]["zk_hosts"] - memory_kb node["redborder"]["memory_services"]["hadoop-nodemanager"]["memory"] - containersMemory node["redborder"]["hadoop"]["containersMemory"] +hadoop_common 'Configure hadoop common resources' do + name node['hostname'] + zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] + memory_kb node['redborder']['memory_services']['hadoop-nodemanager']['memory'] + containersMemory node['redborder']['hadoop']['containersMemory'] + if manager_services['hadoop-nodemanager'] || manager_services['hadoop-resourcemanager'] action :add - end -else - hadoop_common "Delete hadoop common resources" do + else action :remove end end -hadoop_resourcemanager "Configure Hadoop ResourceManager" do - memory_kb node["redborder"]["memory_services"]["hadoop-resourcemanager"]["memory"] - action (manager_services["hadoop-resourcemanager"] ? [:add, :register] : [:remove, :deregister]) +hadoop_resourcemanager 'Configure Hadoop ResourceManager' do + memory_kb node['redborder']['memory_services']['hadoop-resourcemanager']['memory'] + if manager_services['hadoop-resourcemanager'] + action [:add, :register] + else + action [:remove, :deregister] + end end -hadoop_nodemanager "Configure Hadoop NodeManager" do - memory_kb node["redborder"]["memory_services"]["hadoop-nodemanager"]["memory"] - action (manager_services["hadoop-nodemanager"] ? [:add, :register] : [:remove, :deregister]) +hadoop_nodemanager 'Configure Hadoop NodeManager' do + memory_kb node['redborder']['memory_services']['hadoop-nodemanager']['memory'] + if manager_services['hadoop-nodemanager'] + action [:add, :register] + else + action [:remove, :deregister] + end end -samza_config "Configure samza applications" do - memory_per_container node["redborder"]["samza"]["memory_per_container"] - num_containers node["redborder"]["samza"]["num_containers"] - action (manager_services["hadoop-nodemanager"] ? :add : :remove) +samza_config 'Configure samza applications' do + memory_per_container node['redborder']['samza']['memory_per_container'] + num_containers node['redborder']['samza']['num_containers'] + if manager_services['hadoop-nodemanager'] + action :add + else + action :remove + end end -geoip_config "Configure GeoIP" do +geoip_config 'Configure GeoIP' do action :add end -snmp_config "Configure snmp" do - hostname node["hostname"] - cdomain node["redborder"]["cdomain"] - action (manager_services["snmp"] ? :add : :remove) +snmp_config 'Configure snmp' do + hostname node['hostname'] + cdomain node['redborder']['cdomain'] + if manager_services['snmp'] + action :add + else + action :remove + end end -rbmonitor_config "Configure redborder-monitor" do - name node["hostname"] - device_nodes node.run_state["sensors_info_all"]["device-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] - managers node["redborder"]["managers_list"] - cluster node["redborder"]["cluster_info"] - hostip node["redborder"]["cluster_info"][name]["ip"] - action (manager_services["redborder-monitor"] ? :add : :remove) +rbmonitor_config 'Configure redborder-monitor' do + name node['hostname'] + device_nodes node.run_state['sensors_info_all']['device-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + managers node['redborder']['managers_list'] + cluster node['redborder']['cluster_info'] + hostip node['redborder']['cluster_info'][name]['ip'] + if manager_services['redborder-monitor'] + action :add + else + action :remove + end end -rbscanner_config "Configure redborder-scanner" do - scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] - action (manager_services["redborder-scanner"] ? [:add, :register] : [:remove, :deregister]) +rbscanner_config 'Configure redborder-scanner' do + scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] + if manager_services['redborder-scanner'] + action [:add, :register] + else + action [:remove, :deregister] + end end -nginx_config "Configure Nginx" do - cdomain node["redborder"]["cdomain"] - action (manager_services["nginx"] ? [:add, :register] : [:remove, :deregister]) +nginx_config 'Configure Nginx' do + cdomain node['redborder']['cdomain'] + if manager_services['nginx'] + action [:add, :register] + else + action [:remove, :deregister] + end end -if manager_services["nginx"] and manager_services["chef-server"] - nginx_config "Configure Nginx Chef" do - service_name "erchef" - cdomain node["redborder"]["cdomain"] +nginx_config 'Configure Nginx Chef' do + service_name 'erchef' + cdomain node['redborder']['cdomain'] + if manager_services['nginx'] && manager_services['chef-server'] action [:configure_certs, :add_erchef] + else + action :nothing end end -if manager_services["nginx"] and manager_services["rb-aioutliers"] - nginx_config "Configure Nginx aioutliers" do - service_name "rb-aioutliers" - cdomain node["redborder"]["cdomain"] +nginx_config 'Configure Nginx aioutliers' do + service_name 'rb-aioutliers' + cdomain node['redborder']['cdomain'] + if manager_services['nginx'] && manager_services['rb-aioutliers'] action [:configure_certs, :add_aioutliers] + else + action :nothing end end -webui_config "Configure WebUI" do - hostname node["hostname"] - memory_kb node["redborder"]["memory_services"]["webui"]["memory"] - cdomain node["redborder"]["cdomain"] - port node["redborder"]["webui"]["port"] - action (manager_services["webui"] ? [:add, :register, :configure_rsa] : [:remove, :deregister]) +webui_config 'Configure WebUI' do + hostname node['hostname'] + memory_kb node['redborder']['memory_services']['webui']['memory'] + cdomain node['redborder']['cdomain'] + port node['redborder']['webui']['port'] + if manager_services['webui'] + action [:add, :register, :configure_rsa] + else + action [:remove, :deregister] + end end -webui_config "Configure Nginx WebUI" do - cdomain node["redborder"]["cdomain"] - port node["redborder"]["webui"]["port"] - action ((manager_services["webui"] and manager_services["nginx"]) ? [:configure_certs, :add_webui_conf_nginx] : :nothing) +webui_config 'Configure Nginx WebUI' do + cdomain node['redborder']['cdomain'] + port node['redborder']['webui']['port'] + if manager_services['webui'] && manager_services['nginx'] + action [:configure_certs, :add_webui_conf_nginx] + else + action :nothing + end end -http2k_config "Configure Http2k" do - domain node["redborder"]["cdomain"] - kafka_hosts node["redborder"]["managers_per_services"]["kafka"] - memory node["redborder"]["memory_services"]["http2k"]["memory"] - port node["redborder"]["http2k"]["port"] - proxy_nodes node.run_state["sensors_info"]["proxy-sensor"] - ips_nodes node.run_state["sensors_info"]["ips-sensor"] - ipsg_nodes node.run_state["sensors_info"]["ipsg-sensor"] - ipscp_nodes node.run_state["sensors_info"]["ipscp-sensor"] - organizations node["redborder"]["organizations"] - locations_list node["redborder"]["locations"] - action (manager_services["http2k"] ? [:add, :register] : [:remove, :deregister]) +http2k_config 'Configure Http2k' do + domain node['redborder']['cdomain'] + kafka_hosts node['redborder']['managers_per_services']['kafka'] + memory node['redborder']['memory_services']['http2k']['memory'] + port node['redborder']['http2k']['port'] + proxy_nodes node.run_state['sensors_info']['proxy-sensor'] + ips_nodes node.run_state['sensors_info']['ips-sensor'] + ipsg_nodes node.run_state['sensors_info']['ipsg-sensor'] + ipscp_nodes node.run_state['sensors_info']['ipscp-sensor'] + organizations node['redborder']['organizations'] + locations_list node['redborder']['locations'] + if manager_services['http2k'] + action [:add, :register] + else + action [:remove, :deregister] + end end -http2k_config "Configure Nginx Http2k" do - domain node["redborder"]["cdomain"] - port node["redborder"]["http2k"]["port"] - action ((manager_services["http2k"] and manager_services["nginx"]) ? [:configure_certs, :add_http2k_conf_nginx] : :nothing) +http2k_config 'Configure Nginx Http2k' do + domain node['redborder']['cdomain'] + port node['redborder']['http2k']['port'] + if manager_services['http2k'] && manager_services['nginx'] + action [:configure_certs, :add_http2k_conf_nginx] + else + action :nothing + end end -f2k_config "Configure f2k" do - sensors node.run_state["sensors_info"]["flow-sensor"] - action (manager_services["f2k"] ? [:add, :register] : [:remove, :deregister]) +f2k_config 'Configure f2k' do + sensors node.run_state['sensors_info']['flow-sensor'] + if manager_services['f2k'] + action [:add, :register] + else + action [:remove, :deregister] + end end -pmacct_config "Configure pmacct" do - sensors node.run_state["sensors_info"]["flow-sensor"] - kafka_hosts node["redborder"]["managers_per_services"]["kafka"] - action (manager_services["pmacct"] ? [:add, :register] : [:remove, :deregister]) +pmacct_config 'Configure pmacct' do + sensors node.run_state['sensors_info']['flow-sensor'] + kafka_hosts node['redborder']['managers_per_services']['kafka'] + if manager_services['pmacct'] + action [:add, :register] + else + action [:remove, :deregister] + end end -if node.run_state["pipelines"].nil? || node.run_state["pipelines"].empty? - service 'logstash' do - action [:disable, :stop] - end -else - logstash_config "Configure logstash" do - cdomain node["redborder"]["cdomain"] - flow_nodes node.run_state["all_flow_sensors_info"]["flow-sensor"] - namespaces node.run_state["namespaces"] - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] - scanner_nodes node.run_state["sensors_info_all"]["scanner-sensor"] - device_nodes node.run_state["sensors_info_all"]["device-sensor"] - logstash_pipelines node.run_state["pipelines"] - action (manager_services["logstash"] ? [:add, :register] : [:remove, :deregister]) +logstash_config 'Configure logstash' do + cdomain node['redborder']['cdomain'] + flow_nodes node.run_state['all_flow_sensors_info']['flow-sensor'] + namespaces node.run_state['namespaces'] + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] + device_nodes node.run_state['sensors_info_all']['device-sensor'] + logstash_pipelines node.run_state['pipelines'] + if manager_services['logstash'] && node.run_state['pipelines'] && !node.run_state['pipelines'].empty? + action [:add, :register] + else + action [:remove, :deregister] end end -rbdswatcher_config "Configure redborder-dswatcher" do - cdomain node["redborder"]["cdomain"] - action (manager_services["redborder-dswatcher"] ? [:add, :register] : [:remove, :deregister]) +rbdswatcher_config 'Configure redborder-dswatcher' do + cdomain node['redborder']['cdomain'] + if manager_services['redborder-dswatcher'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rbevents_counter_config "Configure redborder-events-counter" do - cdomain node["redborder"]["cdomain"] - action (manager_services["redborder-events-counter"] ? [:add, :register] : [:remove, :deregister]) +rbevents_counter_config 'Configure redborder-events-counter' do + cdomain node['redborder']['cdomain'] + if manager_services['redborder-events-counter'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rsyslog_config "Configure rsyslog" do - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] + node.run_state["sensors_info_all"]["cep-sensor"] - ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] - action (manager_services["rsyslog"] ? [:add, :register] : [:remove, :deregister]) +rsyslog_config 'Configure rsyslog' do + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + node.run_state['sensors_info_all']['cep-sensor'] + ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] + if manager_services['rsyslog'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rbnmsp_config "Configure redborder-nmsp" do - memory node["redborder"]["memory_services"]["redborder-nmsp"]["memory"] - proxy_nodes node.run_state["sensors_info_all"]["proxy-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] - hosts node["redborder"]["zookeeper"]["zk_hosts"] - action (manager_services["redborder-nmsp"] ? [:add, :configure_keys, :register] : [:remove, :deregister]) +rbnmsp_config 'Configure redborder-nmsp' do + memory node['redborder']['memory_services']['redborder-nmsp']['memory'] + proxy_nodes node.run_state['sensors_info_all']['proxy-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + hosts node['redborder']['zookeeper']['zk_hosts'] + if manager_services['redborder-nmsp'] + action [:add, :configure_keys, :register] + else + action [:remove, :deregister] + end end -n2klocd_config "Configure n2klocd" do - mse_nodes node.run_state["sensors_info_all"]["mse-sensor"] - meraki_nodes node.run_state["sensors_info_all"]["meraki-sensor"] - n2klocd_managers node["redborder"]["managers_per_services"]["n2klocd"] - memory node["redborder"]["memory_services"]["n2klocd"]["memory"] - action (manager_services["n2klocd"] ? [:add, :register] : [:remove, :deregister]) +n2klocd_config 'Configure n2klocd' do + mse_nodes node.run_state['sensors_info_all']['mse-sensor'] + meraki_nodes node.run_state['sensors_info_all']['meraki-sensor'] + n2klocd_managers node['redborder']['managers_per_services']['n2klocd'] + memory node['redborder']['memory_services']['n2klocd']['memory'] + if manager_services['n2klocd'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rbale_config "Configure redborder-ale" do - ale_nodes node.run_state["sensors_info_all"]["ale-sensor"] - action (node["redborder"]["services"]["redborder-ale"] ? [:add, :register] : [:remove, :deregister]) +rbale_config 'Configure redborder-ale' do + ale_nodes node.run_state['sensors_info_all']['ale-sensor'] + if node['redborder']['services']['redborder-ale'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rblogstatter_config "Configure redborder-logstatter" do - action (node["redborder"]["services"]["rb-logstatter"] ? :add : :remove) +rblogstatter_config 'Configure redborder-logstatter' do + if node['redborder']['services']['rb-logstatter'] + action :add + else + action :remove + end end -rb_arubacentral_config "Configure rb-arubacentral" do - arubacentral_nodes node.run_state["sensors_info_all"]["arubacentral-sensor"] - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] - action (node["redborder"]["services"]["rb-arubacentral"] ? :add : :remove) +rb_arubacentral_config 'Configure rb-arubacentral' do + arubacentral_nodes node.run_state['sensors_info_all']['arubacentral-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + if node['redborder']['services']['rb-arubacentral'] + action :add + else + action :remove + end end -#freeradius_config "Configure radiusd" do -# flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] -# action (node["redborder"]["services"]["radiusd"] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) -#end +# freeradius_config 'Configure radiusd' do +# flow_nodes node.run_state['sensors_info_all']['flow-sensor'] +# action (node['redborder']['services']['radiusd'] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) +# end -rbaioutliers_config "Configure rb-aioutliers" do - action (manager_services["rb-aioutliers"] ? [:add, :register] : [:remove, :deregister]) +rbaioutliers_config 'Configure rb-aioutliers' do + if manager_services['rb-aioutliers'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rbcep_config "Configure redborder-cep" do - flow_nodes node.run_state["sensors_info_all"]["flow-sensor"] - vault_nodes node.run_state["sensors_info_all"]["vault-sensor"] - ips_nodes node.run_state["sensors_info_all"]["ips-sensor"] + node.run_state["sensors_info_all"]["ipsv2-sensor"] + node.run_state["sensors_info_all"]["ipscp-sensor"] - action (node["redborder"]["services"]["redborder-cep"] ? [:add, :register] : [:remove, :deregister]) +rbcep_config 'Configure redborder-cep' do + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] + if node['redborder']['services']['redborder-cep'] + action [:add, :register] + else + action [:remove, :deregister] + end end -rb_postfix_config "Configure postfix" do - action (node["redborder"]["services"]["postfix"] ? :add : :remove) +rb_postfix_config 'Configure postfix' do + if node['redborder']['services']['postfix'] + action :add + else + action :remove + end end -rbcgroup_config "Configure cgroups" do +rbcgroup_config 'Configure cgroups' do action :add end # Determine external -external_services = Chef::DataBagItem.load("rBglobal", "external_services") +begin + external_services = data_bag_item('rBglobal', 'external_services') +rescue + external_services = {} +end -postgresql_config "Configure postgresql" do - cdomain node["redborder"]["cdomain"] - ipaddress node["ipaddress_sync"] - action (manager_services["postgresql"] and external_services["postgresql"] == "onpremise" ? [:add, :register] : [:remove, :deregister]) +postgresql_config 'Configure postgresql' do + cdomain node['redborder']['cdomain'] + ipaddress node['ipaddress_sync'] + if manager_services['postgresql'] && external_services['postgresql'] == 'onpremise' + action [:add, :register] + else + action [:remove, :deregister] + end end s3_leader = `serf members | grep s3=ready | awk '{print $1'} | head -n 1`.strip # Allow only one s3 onpremise node for now.. TODO: Distributed MinIO -minio_config "Configure S3 (minio)" do - ipaddress node["ipaddress_sync"] - action ((manager_services["s3"] and external_services["s3"] == "onpremise" and s3_leader == node.name ) ? [:add, :register] : [:remove, :deregister]) +minio_config 'Configure S3 (minio)' do + ipaddress node['ipaddress_sync'] + if manager_services['s3'] && external_services['s3'] == 'onpremise' && s3_leader == node.name + action [:add, :register] + else + action [:remove, :deregister] + end end -if manager_services["s3"] - nginx_config "Configure S3 certs" do - service_name "s3" - cdomain node["redborder"]["cdomain"] +nginx_config 'Configure S3 certs' do + service_name 's3' + cdomain node['redborder']['cdomain'] + if manager_services['s3'] action :configure_certs + else + action :nothing end end -ssh_secrets = Chef::DataBagItem.load("passwords", "ssh") rescue ssh_secrets = {} +begin + ssh_secrets = data_bag_item('passwords', 'ssh') +rescue + ssh_secrets = {} +end -directory "/root/.ssh" do - owner "root" - group "root" - mode 0755 +directory '/root/.ssh' do + owner 'root' + group 'root' + mode '0755' action :create end -if !ssh_secrets.empty? - template "/root/.ssh/authorized_keys" do - source "rsa.pub.erb" - owner "root" - group "root" - mode 0600 +unless ssh_secrets.empty? + template '/root/.ssh/authorized_keys' do + source 'rsa.pub.erb' + owner 'root' + group 'root' + mode '0600' retries 2 - variables(:public_rsa => ssh_secrets['public_rsa']) + variables(public_rsa: ssh_secrets['public_rsa']) end end - -#--------------------------SUDOERS--------------------------# - -template "/etc/sudoers.d/redborder-manager" do - source "redborder-manager.erb" - owner "root" - group "root" - mode 0440 +# Sudoers +template '/etc/sudoers.d/redborder-manager' do + source 'redborder-manager.erb' + owner 'root' + group 'root' + mode '0440' retries 2 end -#--------------------------Pending_changes--------------------------# +# Pending Changes.. # pending_changes==0 -> has changes to apply at next chef-client run # pending_changes==1 -> chef-client has to run once # pending_changes==2 -> chef-client has to run twice # ....... # pending_changes==n -> chef-client has to run n times # +node.normal['redborder']['pending_changes'] = node['redborder']['pending_changes'] > 0 ? node.normal['redborder']['pending_changes'].to_i - 1 : 0 -if node["redborder"]["pending_changes"]>0 - node.normal["redborder"]["pending_changes"] = (node.normal["redborder"]["pending_changes"].to_i-1) -else - node.normal["redborder"]["pending_changes"] = 0 -end - -execute "force_chef_client_wakeup" do - command "/usr/lib/redborder/bin/rb_wakeup_chef.sh" +execute 'force_chef_client_wakeup' do + command '/usr/lib/redborder/bin/rb_wakeup_chef.sh' ignore_failure true - action ( node["redborder"]["pending_changes"].nil? or node["redborder"]["pending_changes"]==0 ) ? :nothing : :run + if node['redborder']['pending_changes'].nil? || node['redborder']['pending_changes'] == 0 + action :nothing + else + action :run + end end -#--------------------------MOTD--------------------------# +# MOTD +cluster_info = node['redborder']['cluster_info'] -cluster_info = node["redborder"]["cluster_info"] -cluster_uuid_db = Chef::DataBagItem.load("rBglobal", "cluster") rescue cluster_uuid_db = {} -cluster_installed = File.exist?("/etc/redborder/cluster-installed.txt") +begin + cluster_uuid_db = data_bag_item('rBglobal', 'cluster') +rescue + cluster_uuid_db = {} +end -template "/etc/motd" do - source "motd.erb" - owner "root" - group "root" - mode 0644 - retries 2 - backup false - variables(:cluster_info => cluster_info, :uuid => cluster_uuid_db["uuid"], :manager_services => manager_services, :cluster_finished => cluster_installed) +cluster_installed = File.exist?('/etc/redborder/cluster-installed.txt') + +template '/etc/motd' do + source 'motd.erb' + owner 'root' + group 'root' + mode '0644' + retries 2 + backup false + variables(cluster_info: cluster_info, + uuid: cluster_uuid_db['uuid'], + manager_services: manager_services, + cluster_finished: cluster_installed) end diff --git a/resources/recipes/configure_cron_tasks.rb b/resources/recipes/configure_cron_tasks.rb index 9abaa28d..a7777140 100644 --- a/resources/recipes/configure_cron_tasks.rb +++ b/resources/recipes/configure_cron_tasks.rb @@ -1,29 +1,28 @@ -# -# Cookbook Name:: manager +# Cookbook:: manager # Recipe:: configure_cron_tasks -# -# Copyright 2024, redborder -# -# AFFERO GENERAL PUBLIC LICENSE V3 -# +# Copyright:: 2024, redborder +# License:: Affero General Public License, Version 3 # Services configuration manager_services = manager_services() -cluster_installed = File.exist?("/etc/cluster-installed.txt") +cluster_installed = File.exist?('/etc/cluster-installed.txt') -#--------------------------Druid-------------------------# +# Druid cron_d 'clean_segments_daily' do - action ((manager_services["druid-coordinator"] or manager_services["druid-historical"]) and cluster_installed) ? :create : :delete + if (manager_services['druid-coordinator'] || manager_services['druid-historical']) && cluster_installed + action :create + else + action :delete + end minute '00' hour '01' weekday '*' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_clean_segments.sh" + command '/usr/lib/redborder/bin/rb_clean_segments.sh' end - cron_d 'create_druid_metadata_daily' do action :create minute '00' @@ -31,10 +30,10 @@ weekday '*' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_create_druid_metadata.sh" + command '/usr/lib/redborder/bin/rb_create_druid_metadata.sh' end -#--------------------------REPOS-------------------------# +# Repos cron_d 'repo_updates_daily' do action :create minute '00' @@ -42,23 +41,26 @@ weekday '*' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_repo_updates.sh" + command '/usr/lib/redborder/bin/rb_repo_updates.sh' end -#--------------------------AWS-------------------------# # AWS Cloudwatch needs to be integrated cron_d 'awsmon_hourly' do - action (!node["redborder"].nil? and !node["redborder"]["dmidecode"].nil? and !node["redborder"]["dmidecode"]["manufacturer"].nil? and node["redborder"]["iscloud"] and manager_services["awslogs"]) ? :create : :delete + if node['redborder'] && node['redborder']['dmidecode'] && node['redborder']['dmidecode']['manufacturer'] && node['redborder']['iscloud'] && manager_services['awslogs'] + action :create + else + action :delete + end minute '5' hour '*' weekday '*' retries 2 ignore_failure true - environment({'MEM' => node["filesystem"].select {|k,v| k.start_with?"/dev/mapper/"}.map{|k,v| "--disk-path=#{v["mount"]}"}.join(" ") }) + environment({ 'MEM': node['filesystem'].select { |k, _v| k.start_with?('/dev/mapper/') }.map { |_k, v| "--disk-path=#{v['mount']}" }.join(' ') }) command '/usr/lib/redborder/bin/rb_awsmon.sh --mem-util $MEM --disk-space-util --from-cron --auto-scaling' end -#--------------------------Events-counter-------------------------# +# Events counter cron_d 'eventscounter_daily' do action :create minute '00' @@ -69,7 +71,7 @@ command 'systemctl restart redborder-events-counter &>/dev/null; exit 0;' end -#--------------------------Licenses-------------------------# +# Licenses cron_d 'check_licences_daily' do action :create minute '00' @@ -77,7 +79,7 @@ weekday '*' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_check_licenses_daily.sh" + command '/usr/lib/redborder/bin/rb_check_licenses_daily.sh' end cron_d 'check_licences_weekly' do @@ -87,13 +89,12 @@ weekday '1' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_check_licenses_weekly.sh" + command '/usr/lib/redborder/bin/rb_check_licenses_weekly.sh' end - -#--------------------------Darklist-------------------------# -# TODO Only the master node should have these cron jobs -# if (manager_mode == "master") +# Darklist +# TODO: Only the master node should have these cron jobs +# if (manager_mode == 'master') cron_d 'rb_update_darklist_weekly' do action :create minute '00' @@ -101,7 +102,7 @@ weekday '1' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_update_darklist.sh" + command '/usr/lib/redborder/bin/rb_update_darklist.sh' notifies :run, 'execute[populate_darklist]', :delayed end @@ -112,7 +113,7 @@ weekday '*' retries 2 ignore_failure true - command "/usr/lib/redborder/bin/rb_refresh_darklist_memcached_keys.sh" + command '/usr/lib/redborder/bin/rb_refresh_darklist_memcached_keys.sh' end execute 'populate_darklist' do diff --git a/resources/recipes/configure_journald.rb b/resources/recipes/configure_journald.rb index c9fd2181..67f242d6 100644 --- a/resources/recipes/configure_journald.rb +++ b/resources/recipes/configure_journald.rb @@ -1,24 +1,25 @@ -# -# Cookbook Name:: manager +# Cookbook:: manager # Recipe:: configure_journald -# -# Copyright 2024, redborder -# -# AFFERO GENERAL PUBLIC LICENSE V3 -# +# Copyright:: 2024, redborder +# License:: Affero General Public License, Version 3 -storage = node["redborder"]["manager"]["journald"]["storage"] rescue nil -template "/etc/systemd/journald.conf" do - source "systemd-journald_journald.conf.erb" - owner "root" - group "root" - mode 0440 - retries 2 - notifies :restart, 'service[systemd-journald]', :delayed - variables(:storage => storage) +begin + storage = node['redborder']['manager']['journald']['storage'] +rescue + storage = nil +end + +template '/etc/systemd/journald.conf' do + source 'systemd-journald_journald.conf.erb' + owner 'root' + group 'root' + mode '0440' + retries 2 + notifies :restart, 'service[systemd-journald]', :delayed + variables(storage: storage) end service 'systemd-journald' do - supports :status => true, :start => true, :restart => true, :reload => true - action :nothing + supports status: true, start: true, restart: true, reload: true + action :nothing end diff --git a/resources/recipes/default.rb b/resources/recipes/default.rb index 92b7a56d..960ca6c3 100644 --- a/resources/recipes/default.rb +++ b/resources/recipes/default.rb @@ -1,11 +1,7 @@ -# -# Cookbook Name:: manager +# Cookbook:: manager # Recipe:: default -# -# Copyright 2016, redborder -# -# AFFERO GENERAL PUBLIC LICENSE V3 -# +# Copyright:: 2024, redborder +# License:: Affero General Public License, Version 3 include_recipe 'rb-manager::prepare_system' include_recipe 'rb-manager::configure' diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 7551a7d6..2b65c1be 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -1,149 +1,147 @@ -# -# Cookbook Name:: manager +# Cookbook:: manager # Recipe:: prepare_system -# -# Copyright 2016, redborder -# -# AFFERO GENERAL PUBLIC LICENSE V3 -# -extend Rb_manager::Helpers - -#clean metadata to get packages upgrades -execute "Clean yum metadata" do - command "yum clean metadata" +# Copyright:: 2024, redborder +# License:: Affero General Public License, Version 3 + +extend RbManager::Helpers + +# clean metadata to get packages upgrades +execute 'Clean yum metadata' do + command 'yum clean metadata' end # Set services_group related with the node mode (core, full, ...) -mode = node["redborder"]["mode"] -node["redborder"]["services_group"][mode].each do |s| - node.default["redborder"]["services"][s] = true -end -if mode != "core" or mode != "full" - node.default["redborder"]["services"]["consul-client"] = true +mode = node['redborder']['mode'] +node['redborder']['services_group'][mode].each { |s| node.default['redborder']['services'][s] = true } + +if mode != 'core' || mode != 'full' + node.default['redborder']['services']['consul-client'] = true end -#Set :ipaddress_sync -ipaddress_sync=node["ipaddress"] +# Set :ipaddress_sync +ipaddress_sync = node['ipaddress'] sync_net = `cat /etc/redborder/rb_init_conf.yml | grep sync_net | awk '{print $2'} | sed 's|/.*||'`.strip -node['network']['interfaces'].each do |interface, details| - next unless "x#{interface}" != "xlo" - ipaddress_sync = `ip route get #{sync_net} | head -n 1 | awk '{for (i=1; i<=NF; i++) if ($i == "src") print $(i+1)}'`.strip + +node['network']['interfaces'].each do |interface, _details| + next unless "x#{interface}" != 'xlo' + + # ipaddress_sync = `ip route get #{sync_net} | head -n 1 | awk '{for (i=1; i<=NF; i++) if ($i == 'src') print $(i+1)}'`.strip + ipaddress_sync = `ip route get #{sync_net} | head -n 1 | awk '/src/ {print $5}'`.strip end -node.default[:ipaddress_sync]=ipaddress_sync -#get mac +node.default[:ipaddress_sync] = ipaddress_sync + +# get mac mac_sync = `ip a | grep -w -B2 #{ipaddress_sync} | awk '{print toupper($2)}' | head -n 1 | tr -d '\n'` -node.default["mac_sync"] = mac_sync +node.default['mac_sync'] = mac_sync -#Configure and enable chef-client -dnf_package "redborder-chef-client" do +# Configure and enable chef-client +dnf_package 'redborder-chef-client' do flush_cache [:before] action :upgrade end -template "/etc/sysconfig/chef-client" do - source "sysconfig_chef-client.rb" - mode 0644 - variables( - :interval => node["chef-client"]["interval"], - :splay => node["chef-client"]["splay"], - :options => node["chef-client"]["options"] - ) +template '/etc/sysconfig/chef-client' do + source 'sysconfig_chef-client.rb.erb' + mode '0644' + variables(interval: node['chef-client']['interval'], + splay: node['chef-client']['splay'], + options: node['chef-client']['options']) end -if node["redborder"]["services"]["chef-client"] - service "chef-client" do +service 'chef-client' do + if node['redborder']['services']['chef-client'] action [:enable, :start] - end -else - service "chef-client" do + else action [:stop] end end -#get managers information(name, ip, services...) -cdomain = "" -File.open('/etc/redborder/cdomain') {|f| cdomain = f.readline.chomp} -node.default["redborder"]["cdomain"] = cdomain +# get managers information(name, ip, services...) +cdomain = '' +File.open('/etc/redborder/cdomain') { |f| cdomain = f.readline.chomp } +node.default['redborder']['cdomain'] = cdomain -#get managers information(name, ip, services...) -node.default["redborder"]["cluster_info"] = get_cluster_info() +# get managers information(name, ip, services...) +node.default['redborder']['cluster_info'] = get_cluster_info() -#get managers sorted by service -node.default["redborder"]["managers_per_services"] = managers_per_service() +# get managers sorted by service +node.default['redborder']['managers_per_services'] = managers_per_service() + +# get elasticache nodes +begin + elasticache = data_bag_item('rBglobal', 'elasticache') +rescue + elasticache = {} +end -#get elasticache nodes -elasticache = Chef::DataBagItem.load("rBglobal", "elasticache") rescue elasticache = {} if !elasticache.empty? - node.default["redborder"]["memcached"]["server_list"] = getElasticacheNodes(elasticache["cfg_address"], elasticache["cfg_port"]) - node.default["redborder"]["memcached"]["port"] = elasticache["cfg_port"] - node.default["redborder"]["memcached"]["hosts"] = joinHostArray2port(node["redborder"]["memcached"]["server_list"], node["redborder"]["memcached"]["port"]).join(",") - node.default["redborder"]["memcached"]["elasticache"] = true + node.default['redborder']['memcached']['server_list'] = getElasticacheNodes(elasticache['cfg_address'], elasticache['cfg_port']) + node.default['redborder']['memcached']['port'] = elasticache['cfg_port'] + node.default['redborder']['memcached']['hosts'] = joinHostArray2port(node['redborder']['memcached']['server_list'], node['redborder']['memcached']['port']).join(',') + node.default['redborder']['memcached']['elasticache'] = true else - node.default["redborder"]["memcached"]["hosts"] = "memcached.service.#{node["redborder"]["cdomain"]}:#{node["redborder"]["memcached"]["port"]}" + node.default['redborder']['memcached']['hosts'] = "memcached.service.#{node['redborder']['cdomain']}:#{node['redborder']['memcached']['port']}" end -#get organizations for http2k -node.default["redborder"]["organizations"] = get_orgs() if node["redborder"]["services"]["http2k"] +# get organizations for http2k +node.default['redborder']['organizations'] = get_orgs() if node['redborder']['services']['http2k'] -#get sensors info -node.run_state["sensors_info"] = get_sensors_info() +# get sensors info +node.run_state['sensors_info'] = get_sensors_info() -#get sensors info full info -node.run_state["sensors_info_all"] = get_sensors_all_info() +# get sensors info full info +node.run_state['sensors_info_all'] = get_sensors_all_info() -#get sensors info of all flow sensors -node.run_state["all_flow_sensors_info"] = get_all_flow_sensors_info() +# get sensors info of all flow sensors +node.run_state['all_flow_sensors_info'] = get_all_flow_sensors_info() -#get logstash pipelines -node.run_state["pipelines"] = get_pipelines() +# get logstash pipelines +node.run_state['pipelines'] = get_pipelines() -#get namespaces -node.run_state["namespaces"] = get_namespaces +# get namespaces +node.run_state['namespaces'] = get_namespaces -#get string with all zookeeper hosts and port separated by commas, its needed for multiples services -zk_port = node["redborder"]["zookeeper"]["port"] -#zk_hosts = node["redborder"]["managers_per_services"]["zookeeper"].map {|z| "#{z}.node:#{zk_port}"}.join(',') -node.default["redborder"]["zookeeper"]["zk_hosts"] = "zookeeper.service.#{node["redborder"]["cdomain"]}:#{node["redborder"]["zookeeper"]["port"]}" +# get string with all zookeeper hosts and port separated by commas, its needed for multiples services +# zk_port = node['redborder']['zookeeper']['port'] +# zk_hosts = node['redborder']['managers_per_services']['zookeeper'].map {|z| '#{z}.node:#{zk_port}'}.join(',') +node.default['redborder']['zookeeper']['zk_hosts'] = "zookeeper.service.#{node['redborder']['cdomain']}:#{node['redborder']['zookeeper']['port']}" -#set kafka host index if kafka is enabled in this host -if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) - node.default["redborder"]["kafka"]["host_index"] = node["redborder"]["managers_per_services"]["kafka"].index(node.name) +# set kafka host index if kafka is enabled in this host +if node['redborder']['managers_per_services']['kafka'].include?(node.name) + node.default['redborder']['kafka']['host_index'] = node['redborder']['managers_per_services']['kafka'].index(node.name) end -#set druid realtime partition id (its needed in cluster mode for druid brokers) -if node["redborder"]["managers_per_services"]["druid-realtime"].include?(node.name) - node.default["redborder"]["druid"]["realtime"]["partition_num"] = node["redborder"]["managers_per_services"]["druid-realtime"].index(node.name) +# set druid realtime partition id (its needed in cluster mode for druid brokers) +if node['redborder']['managers_per_services']['druid-realtime'].include?(node.name) + node.default['redborder']['druid']['realtime']['partition_num'] = node['redborder']['managers_per_services']['druid-realtime'].index(node.name) end -#get an array of managers +# get an array of managers managers_list = [] -node["redborder"]["cluster_info"].each_key do |mgr| - managers_list << mgr -end -node.default["redborder"]["managers_list"] = managers_list +node['redborder']['cluster_info'].each_key { |mgr| managers_list << mgr } +node.default['redborder']['managers_list'] = managers_list -#hard disk -node.default["redborder"]["manager"]["hd_services_current"] = harddisk_services() +# hard disk +node.default['redborder']['manager']['hd_services_current'] = harddisk_services() -#memory -#getting total system memory less 10% reserved by system -sysmem_total = (node["memory"]["total"].to_i * 0.90).to_i -#node attributes related with memory are changed inside the function to have simplicity using recursivity +# memory +# getting total system memory less 10% reserved by system +sysmem_total = (node['memory']['total'].to_i * 0.90).to_i +# node attributes related with memory are changed inside the function to have simplicity using recursivity memory_services(sysmem_total) -#License - -modules = ["ips", "flow", "monitor", "location", "api", "malware", "vault"] +# License +modules = %w(ips flow monitor location api malware vault) fmodules = [] modules.each do |x| - if !node["redborder"].nil? and !node["redborder"]["manager"].nil? and !node["redborder"]["manager"]["modules"].nil? and !node["redborder"]["manager"]["modules"][x].nil? - fmodules << x if node["redborder"]["manager"]["modules"][x] + if node['redborder'] && node['redborder']['manager'] && node['redborder']['manager']['modules'] && node['redborder']['manager']['modules'][x] + fmodules << x if node['redborder']['manager']['modules'][x] else fmodules << x end end -node.normal["redborder"]["license"]["fmodules"] = fmodules +node.normal['redborder']['license']['fmodules'] = fmodules diff --git a/resources/templates/default/sysconfig_chef-client.rb b/resources/templates/default/sysconfig_chef-client.rb.erb similarity index 100% rename from resources/templates/default/sysconfig_chef-client.rb rename to resources/templates/default/sysconfig_chef-client.rb.erb From 88c7af829188d7d0b52c68709a4753877e6d3915 Mon Sep 17 00:00:00 2001 From: vimesa Date: Tue, 11 Jun 2024 08:57:19 +0100 Subject: [PATCH 44/50] Release 1.9.3 --- CHANGELOG.md | 5 +++++ resources/metadata.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f96a1d..81f48c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ cookbook-rb-manager CHANGELOG =============== +## 1.9.3 + + - Miguel Negrón + - [a12be85] Improvement/fix lint (#148) + ## 1.9.2 - Miguel Negrón diff --git a/resources/metadata.rb b/resources/metadata.rb index 3ce5b42e..5ac81c79 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,7 +3,7 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures redborder manager' -version '1.9.2' +version '1.9.3' depends 'rb-common' depends 'chef-server' From a28b7289d2d5846f17dacf628cbf423a810c95b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez?= <128592227+malvads@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:08:19 +0100 Subject: [PATCH 45/50] Configure minio nodes and load balancer (#145) * Configure minio nodes and load balancer * Proper use of s3 hosts * Fix prepare and configure * Add s3_init_conf template * Delete resources/templates/default/s3_init_conf.yml.erb * Configure S3 cert before nginx * Remove configure certs from minio nginx * Fix lint issues --- resources/attributes/default.rb | 1 + resources/recipes/configure.rb | 33 ++++++++++++++++----------- resources/recipes/prepare_system.rb | 4 ++++ resources/templates/default/minio.erb | 4 ++++ 4 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 resources/templates/default/minio.erb diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index ff2f11b4..8354d4e6 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -6,6 +6,7 @@ # s3 default['redborder']['uploaded_s3'] = false +default['redborder']['s3']['s3_hosts'] = [] # chef-client default['chef-client']['interval'] = 300 diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index b2c1119a..a7aaa90e 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -510,28 +510,35 @@ end end -s3_leader = `serf members | grep s3=ready | awk '{print $1'} | head -n 1`.strip +begin + s3_secrets = data_bag_item('passwords', 's3') +rescue + ssh_secrets = {} +end -# Allow only one s3 onpremise node for now.. TODO: Distributed MinIO +# Allow only s3 onpremise nodes for now.. minio_config 'Configure S3 (minio)' do ipaddress node['ipaddress_sync'] - if manager_services['s3'] && external_services['s3'] == 'onpremise' && s3_leader == node.name - action [:add, :register] - else - action [:remove, :deregister] - end + access_key_id s3_secrets['s3_access_key_id'] + secret_key_id s3_secrets['s3_secret_key_id'] + action((manager_services['s3'] && (external_services['s3'] == 'onpremise')) ? [:add, :register] : [:remove, :deregister]) end -nginx_config 'Configure S3 certs' do - service_name 's3' - cdomain node['redborder']['cdomain'] - if manager_services['s3'] +# First configure the cert for the service before configuring nginx +if manager_services['s3'] + nginx_config 'Configure S3 certs' do + service_name 's3' + cdomain node['redborder']['cdomain'] action :configure_certs - else - action :nothing end end +# Configure Nginx s3 onpremise nodes for now.. +minio_config 'Configure Nginx S3 (minio)' do + s3_hosts node['redborder']['s3']['s3_hosts'] + action((manager_services['s3'] && (external_services['s3'] == 'onpremise')) ? [:add_s3_conf_nginx] : :nothing) +end + begin ssh_secrets = data_bag_item('passwords', 'ssh') rescue diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 2b65c1be..dd996fa6 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -107,6 +107,10 @@ # zk_hosts = node['redborder']['managers_per_services']['zookeeper'].map {|z| '#{z}.node:#{zk_port}'}.join(',') node.default['redborder']['zookeeper']['zk_hosts'] = "zookeeper.service.#{node['redborder']['cdomain']}:#{node['redborder']['zookeeper']['port']}" +# Set all nodes with s3 configured (nginx load balancer) +s3_hosts = node['redborder']['managers_per_services']['s3'].map { |z| "#{z}.node:9000" } +node.default['redborder']['s3']['s3_hosts'] = s3_hosts + # set kafka host index if kafka is enabled in this host if node['redborder']['managers_per_services']['kafka'].include?(node.name) node.default['redborder']['kafka']['host_index'] = node['redborder']['managers_per_services']['kafka'].index(node.name) diff --git a/resources/templates/default/minio.erb b/resources/templates/default/minio.erb new file mode 100644 index 00000000..4973ec55 --- /dev/null +++ b/resources/templates/default/minio.erb @@ -0,0 +1,4 @@ +MINIO_OPTS="--address :9000 --console-address :9001 --config-dir /etc/minio" +MINIO_VOLUMES=/var/minio/data +MINIO_ROOT_USER=<%= @s3_user %> +MINIO_ROOT_PASSWORD=<%= @s3_password %> \ No newline at end of file From 13bfb8a38f10423ce5d4dfcd4dfd5bf9b8323bd9 Mon Sep 17 00:00:00 2001 From: David Vanhoucke Date: Thu, 13 Jun 2024 14:11:53 +0100 Subject: [PATCH 46/50] bump to 2.0.0 --- CHANGELOG.md | 5 +++++ resources/metadata.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81f48c6d..3108fac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ cookbook-rb-manager CHANGELOG =============== +## 2.0.0 + + - Miguel Álvarez + - [a28b728] Configure minio nodes and load balancer (#145) + ## 1.9.3 - Miguel Negrón diff --git a/resources/metadata.rb b/resources/metadata.rb index 5ac81c79..ac1f7370 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,7 +3,7 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures redborder manager' -version '1.9.3' +version '2.0.0' depends 'rb-common' depends 'chef-server' From e748afeddc41ac7a367b133ed0912ff9e96915ee Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Thu, 13 Jun 2024 14:26:54 +0100 Subject: [PATCH 47/50] Fix lint issues --- resources/recipes/prepare_system.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index d307168c..d3e6c1d7 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -108,12 +108,12 @@ node.default['redborder']['zookeeper']['zk_hosts'] = "zookeeper.service.#{node['redborder']['cdomain']}:#{node['redborder']['zookeeper']['port']}" # set webui hosts -webui_hosts = node["redborder"]["managers_per_services"]["webui"].map {|z| "#{z}.node"} -node.default["redborder"]["webui"]["hosts"] = webui_hosts +webui_hosts = node['redborder']['managers_per_services']['webui'].map { |z| "#{z}.node" } +node.default['redborder']['webui']['hosts'] = webui_hosts -#set kafka host index if kafka is enabled in this host -if node["redborder"]["managers_per_services"]["kafka"].include?(node.name) - node.default["redborder"]["kafka"]["host_index"] = node["redborder"]["managers_per_services"]["kafka"].index(node.name) +# set kafka host index if kafka is enabled in this host +if node['redborder']['managers_per_services']['kafka'].include?(node.name) + node.default['redborder']['kafka']['host_index'] = node['redborder']['managers_per_services']['kafka'].index(node.name) end # Set all nodes with s3 configured (nginx load balancer) From 240599304b6e39285df19b8ad3421f60c8b2f3cb Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Thu, 13 Jun 2024 14:28:38 +0100 Subject: [PATCH 48/50] Fix lint issues in configure --- resources/recipes/configure.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 2de221b4..149e4ddd 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -307,7 +307,7 @@ end webui_config 'Configure Nginx WebUI' do - hosts node["redborder"]["webui"]["hosts"] + hosts node['redborder']['webui']['hosts'] cdomain node['redborder']['cdomain'] port node['redborder']['webui']['port'] if manager_services['webui'] && manager_services['nginx'] From d6ecac24673a7a7ff4407efdf14447bca583b7a1 Mon Sep 17 00:00:00 2001 From: Miguel Alvarez Date: Thu, 13 Jun 2024 14:30:48 +0100 Subject: [PATCH 49/50] Fix lint issues in attributes --- resources/attributes/default.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 0fcc7d6f..d078d0a5 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -25,9 +25,9 @@ # http2k default['redborder']['http2k']['port'] = 7980 -#webui -default["redborder"]["webui"]["port"] = 8001 -default["redborder"]["webui"]["hosts"] = [] +# webui +default['redborder']['webui']['port'] = 8001 +default['redborder']['webui']['hosts'] = [] # memcached default['redborder']['memcached']['elasticache'] = false From 5283cdaae6e4717d386ff0b953f0108ebbd68a4b Mon Sep 17 00:00:00 2001 From: JuanSheba Date: Thu, 13 Jun 2024 14:48:06 +0100 Subject: [PATCH 50/50] Release 2.0.1 --- CHANGELOG.md | 11 +++++++++++ resources/metadata.rb | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3108fac4..ff344d72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ cookbook-rb-manager CHANGELOG =============== +## 2.0.1 + + - Miguel Alvarez + - [d6ecac2][e748afe][2405993] Fix lint issues + - [292288d] Add hosts in node data + - [fd63ad2] Fix prepare and configure + - [ac54c02] Proper use of s3 hosts + - [f08e447] Configure minio nodes and load balancer + - [78f1562][4db8fa1][e101c2d][3220e71][4ec77f8][d89bd26][ed91ff3] Updates + - [6e7680a] Delete resources/templates/default/minio.erb + ## 2.0.0 - Miguel Álvarez diff --git a/resources/metadata.rb b/resources/metadata.rb index ac1f7370..d872bf9e 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,7 +3,7 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures redborder manager' -version '2.0.0' +version '2.0.1' depends 'rb-common' depends 'chef-server'