From 787d0ae7f005293329e07d22989f6f9cbfae2a1b Mon Sep 17 00:00:00 2001 From: Rafael Gomez Date: Tue, 5 Nov 2024 09:21:16 +0000 Subject: [PATCH 1/9] Passing hd_services_current maxsize to druid-historical --- resources/recipes/configure.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index 869c3158..21c56f34 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -195,6 +195,7 @@ name node['hostname'] ipaddress node['ipaddress_sync'] memory_kb node['redborder']['memory_services']['druid-historical']['memory'] + maxsize node['redborder']['manager']['hd_services_current']['druid_historical'] action [:add, :register] else action [:remove, :deregister] From 55d679c3406111c167e6dffebc1344fae4d67ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=20G=C3=B3mez?= Date: Sat, 9 Nov 2024 10:45:50 +0000 Subject: [PATCH 2/9] First version of harddisk_services NG --- resources/libraries/harddisk_services.rb | 38 +++++------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index 8d15b555..3888b25e 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -3,48 +3,26 @@ module Helpers def harddisk_services hd_services = {} - root_dev = node['redborder']['manager']['data_dev']['root'] - raw_dev = node['redborder']['manager']['data_dev']['raw'] - aggregate_dev = node['redborder']['manager']['data_dev']['aggregate'] + root_dev = node['filesystem']['by_mountpoint']['/'] - root_dev = node['filesystem'].keys.first if node['filesystem'][root_dev].nil? - if node['filesystem'][raw_dev].nil? && node['filesystem'][aggregate_dev].nil? - raw_dev = root_dev - aggregate_dev = root_dev - elsif node['filesystem'][raw_dev].nil? - raw_dev = aggregate_dev - elsif node['filesystem'][aggregate_dev].nil? - aggregate_dev = raw_dev - end - - hd_services_dev = {} - hd_services_dev['root'] = root_dev - hd_services_dev['raw'] = raw_dev - hd_services_dev['aggregate'] = aggregate_dev + hd_services_dev = { 'root' => root_dev, 'raw' => root_dev, 'aggregate' => root_dev } - hd_services_size = {} - hd_services_size[root_dev] = 0 - hd_services_size[raw_dev] = 0 - hd_services_size[aggregate_dev] = 0 - - hd_services_size_total = {} - hd_services_size_total[root_dev] = 0 - hd_services_size_total[raw_dev] = 0 - hd_services_size_total[aggregate_dev] = 0 + hd_services_size = Hash.new(0) + hd_services_size_total = Hash.new(0) maxsize = {} hd_services_dev.each do |type, device| maxsize[type] = if node['filesystem'][device].nil? 300_000_000_000 else - filesystem_data = node['filesystem'][device] - root_device = hd_services_dev['root'] - size_kb = filesystem_data['kb_size'] || node['filesystem'][root_device]['kb_size'] + filesystem_data = root_dev['kb_size'] + size_kb = filesystem_data size_bytes = size_kb.to_i * 1024 - size_bytes -= 7 * 1024 * 1024 * 1024 if device == root_device + size_bytes -= 7 * 1024 * 1024 * 1024 [size_bytes, 0].max end end + node['redborder']['manager']['hd_services'].each do |s| if node['redborder']['services'][s[:name]] hd_services_size[hd_services_dev[s[:prefered]]] += s[:count] From 58a14bf4dee02a4d7817ca59bc4039b6381bf3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=20G=C3=B3mez?= Date: Sat, 9 Nov 2024 13:09:50 +0000 Subject: [PATCH 3/9] Refactor harddisk_services to only use / kb_size --- resources/libraries/harddisk_services.rb | 49 +++++------------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index 3888b25e..b2852797 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -2,48 +2,19 @@ module RbManager module Helpers def harddisk_services hd_services = {} - - root_dev = node['filesystem']['by_mountpoint']['/'] - - hd_services_dev = { 'root' => root_dev, 'raw' => root_dev, 'aggregate' => root_dev } - - hd_services_size = Hash.new(0) - hd_services_size_total = Hash.new(0) - - maxsize = {} - hd_services_dev.each do |type, device| - maxsize[type] = if node['filesystem'][device].nil? - 300_000_000_000 - else - filesystem_data = root_dev['kb_size'] - size_kb = filesystem_data - size_bytes = size_kb.to_i * 1024 - size_bytes -= 7 * 1024 * 1024 * 1024 - [size_bytes, 0].max - end - end - - node['redborder']['manager']['hd_services'].each do |s| - if node['redborder']['services'][s[:name]] - hd_services_size[hd_services_dev[s[:prefered]]] += s[:count] - end - hd_services_size_total[hd_services_dev[s[:prefered]]] += s[:count] - end - + root_dev_bytes = node['filesystem']['by_mountpoint']['/']['kb_size'].to_i * 1024 + node['redborder']['manager']['hd_services'].each do |service| - preferred_device = hd_services_dev[service[:prefered]] - preferred_size = hd_services_size[preferred_device] - total_size = hd_services_size_total[preferred_device] + service_name = service['name'] + service_count = service['count'] - size = if preferred_size > 0 - preferred_size - else - total_size > 0 ? total_size : 1 - end + allocated_bytes = if service_count > 1 + (root_dev_bytes * 0.4).to_f + else + (root_dev_bytes * 0.01).to_f + end - service_count = service[:count].to_i - max_preferred_size = maxsize[service[:prefered]].to_i - hd_services[service[:name]] = ((service_count * max_preferred_size * 0.90) / size.to_f) / (1024 * 1024) + hd_services[service_name] = allocated_bytes end hd_services From 6c396a68df664bcb2bd6773d5229a6270d70a359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=20G=C3=B3mez?= Date: Sat, 9 Nov 2024 16:58:32 +0000 Subject: [PATCH 4/9] Changing count values to represent porcentage --- resources/attributes/default.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 80473d72..165adec2 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -50,10 +50,10 @@ default['redborder']['manager']['data_dev']['raw'] = '/dev/mapper/vg_rbdata-lv_raw' default['redborder']['manager']['data_dev']['aggregate'] = '/dev/mapper/vg_rbdata-lv_aggregated' default['redborder']['manager']['hd_services'] = [ - { 'name': 'kafka', 'count': 5, 'prefered': 'aggregate' }, + { 'name': 'kafka', 'count': 4, 'prefered': 'aggregate' }, { 'name': 'zookeeper', 'count': 1, 'prefered': 'aggregate' }, - { 'name': 'riak', 'count': 50, 'prefered': 'raw' }, - { 'name': 'druid_historical', 'count': 50, 'prefered': 'raw' }, + { 'name': 'riak', 'count': 40, 'prefered': 'raw' }, + { 'name': 'druid_historical', 'count': 40, 'prefered': 'raw' }, ] default['redborder']['manager']['hd_services_current'] = {} From df635bfce31b88cc49ac97b49489bfaf18973a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=20G=C3=B3mez?= Date: Sat, 9 Nov 2024 17:00:04 +0000 Subject: [PATCH 5/9] harddisk_services method updated --- resources/libraries/harddisk_services.rb | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index b2852797..8cafa154 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -3,21 +3,14 @@ module Helpers def harddisk_services hd_services = {} root_dev_bytes = node['filesystem']['by_mountpoint']['/']['kb_size'].to_i * 1024 - node['redborder']['manager']['hd_services'].each do |service| service_name = service['name'] service_count = service['count'] - - allocated_bytes = if service_count > 1 - (root_dev_bytes * 0.4).to_f - else - (root_dev_bytes * 0.01).to_f - end - - hd_services[service_name] = allocated_bytes + porcentage = service_count / 100 + allocated_bytes = root_dev_bytes * porcentage + hd_services[service_name] = allocated_bytes.to_i end - - hd_services + hd_services # hard disk services sizes are pased in bytes end end end From eab6d1ab78ceb823c57fe539774b04146e077899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=20G=C3=B3mez?= Date: Sat, 9 Nov 2024 17:52:34 +0000 Subject: [PATCH 6/9] Refactor allocated_bytes --- resources/libraries/harddisk_services.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index 8cafa154..2eb018f4 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -6,8 +6,7 @@ def harddisk_services node['redborder']['manager']['hd_services'].each do |service| service_name = service['name'] service_count = service['count'] - porcentage = service_count / 100 - allocated_bytes = root_dev_bytes * porcentage + allocated_bytes = root_dev_bytes * (service_count.to_f / 100) hd_services[service_name] = allocated_bytes.to_i end hd_services # hard disk services sizes are pased in bytes From 8edb6169d3c8a0d69c636b8582133bed1eba5f53 Mon Sep 17 00:00:00 2001 From: Rafael Gomez Date: Tue, 12 Nov 2024 11:21:06 +0000 Subject: [PATCH 7/9] Refactor harddisk_services to check active services, decrease root size and 90% of maxsize --- resources/libraries/harddisk_services.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index 2eb018f4..7a4a387f 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -2,13 +2,23 @@ module RbManager module Helpers def harddisk_services hd_services = {} - root_dev_bytes = node['filesystem']['by_mountpoint']['/']['kb_size'].to_i * 1024 - node['redborder']['manager']['hd_services'].each do |service| - service_name = service['name'] - service_count = service['count'] - allocated_bytes = root_dev_bytes * (service_count.to_f / 100) - hd_services[service_name] = allocated_bytes.to_i + root_dev_kb = node['filesystem']['by_mountpoint']['/']['kb_size'] + size_bytes = root_dev_kb.to_i * 1024 + size_bytes -= 7 * 1024 * 1024 * 1024 + maxsize = size_bytes + + hd_services_dev = node['redborder']['manager']['hd_services'].map do |service| + { + name: service[:name], + count: node['redborder']['services'][service[:name]] ? service[:count] + 1 : service[:count] + } end + + hd_services_dev.each do |service| + service_count = service[:count].to_f / 100 + hd_services[service[:name]] = service_count * maxsize * 0.90 + end + hd_services # hard disk services sizes are pased in bytes end end From a50d54f4b158f782b88017bce8a18df882066226 Mon Sep 17 00:00:00 2001 From: Rafael Gomez Date: Tue, 12 Nov 2024 11:21:32 +0000 Subject: [PATCH 8/9] Changing riak service to s3 --- resources/attributes/default.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 165adec2..ad755612 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -50,10 +50,10 @@ default['redborder']['manager']['data_dev']['raw'] = '/dev/mapper/vg_rbdata-lv_raw' default['redborder']['manager']['data_dev']['aggregate'] = '/dev/mapper/vg_rbdata-lv_aggregated' default['redborder']['manager']['hd_services'] = [ - { 'name': 'kafka', 'count': 4, 'prefered': 'aggregate' }, + { 'name': 'kafka', 'count': 5, 'prefered': 'aggregate' }, { 'name': 'zookeeper', 'count': 1, 'prefered': 'aggregate' }, - { 'name': 'riak', 'count': 40, 'prefered': 'raw' }, - { 'name': 'druid_historical', 'count': 40, 'prefered': 'raw' }, + { 'name': 's3', 'count': 50, 'prefered': 'raw' }, + { 'name': 'druid-historical', 'count': 50, 'prefered': 'raw' }, ] default['redborder']['manager']['hd_services_current'] = {} From f422f3101c2abb0183037ce7b37e8ba2ff67b545 Mon Sep 17 00:00:00 2001 From: Rafael Gomez Date: Tue, 12 Nov 2024 11:23:42 +0000 Subject: [PATCH 9/9] Fix linter --- resources/libraries/harddisk_services.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/libraries/harddisk_services.rb b/resources/libraries/harddisk_services.rb index 7a4a387f..16b94fb4 100644 --- a/resources/libraries/harddisk_services.rb +++ b/resources/libraries/harddisk_services.rb @@ -2,7 +2,7 @@ module RbManager module Helpers def harddisk_services hd_services = {} - root_dev_kb = node['filesystem']['by_mountpoint']['/']['kb_size'] + root_dev_kb = node['filesystem']['by_mountpoint']['/']['kb_size'] size_bytes = root_dev_kb.to_i * 1024 size_bytes -= 7 * 1024 * 1024 * 1024 maxsize = size_bytes @@ -10,7 +10,7 @@ def harddisk_services hd_services_dev = node['redborder']['manager']['hd_services'].map do |service| { name: service[:name], - count: node['redborder']['services'][service[:name]] ? service[:count] + 1 : service[:count] + count: node['redborder']['services'][service[:name]] ? service[:count] + 1 : service[:count], } end