diff --git a/resources/attributes/default.rb b/resources/attributes/default.rb index 8d2a6e93..55cc05d9 100644 --- a/resources/attributes/default.rb +++ b/resources/attributes/default.rb @@ -36,11 +36,6 @@ default['redborder']['memcached']['options'] = '' default['redborder']['memcached']['port'] = 11211 -# hadoop -default['redborder']['hadoop']['containersMemory'] = 2048 -# samza -default['redborder']['samza']['num_containers'] = 1 -default['redborder']['samza']['memory_per_container'] = 2560 # riak # redborder-ai @@ -56,7 +51,6 @@ { 'name': 'zookeeper', 'count': 1, 'prefered': 'aggregate' }, { 'name': 'riak', 'count': 50, 'prefered': 'raw' }, { 'name': 'druid_historical', 'count': 50, 'prefered': 'raw' }, - { 'name': 'hadoop_datanode', 'count': 50, 'prefered': 'raw' }, ] default['redborder']['manager']['hd_services_current'] = {} @@ -78,8 +72,6 @@ default['redborder']['memory_services']['redborder-postgresql'] = { 'count': 5, 'memory': 0 } default['redborder']['memory_services']['mongodb'] = { 'count': 10, 'memory': 0 } default['redborder']['memory_services']['memcached'] = { 'count': 10, 'memory': 0 } -default['redborder']['memory_services']['hadoop-nodemanager'] = { 'count': 50, 'memory': 0 } -default['redborder']['memory_services']['hadoop-resourcemanager'] = { 'count': 10, 'memory': 0 } default['redborder']['memory_services']['snmp'] = { 'count': 5, 'memory': 0, 'max_limit': 10000 } default['redborder']['memory_services']['redborder-monitor'] = { 'count': 5, 'memory': 0, 'max_limit': 20000 } default['redborder']['memory_services']['rb-logstatter'] = { 'count': 5, 'memory': 0, 'max_limit': 20000 } @@ -116,14 +108,13 @@ redborder-events-counter http2k redborder-mem2incident) default['redborder']['services_group']['custom'] = [] -default['redborder']['services_group']['core'] = %w(consul zookeeper druid-coordinator druid-overlord hadoop-resourcemanager) # consul server +default['redborder']['services_group']['core'] = %w(consul zookeeper druid-coordinator druid-overlord) # consul server default['redborder']['services_group']['chef'] = %w(chef-server) default['redborder']['services_group']['kafka'] = %w(kafka) default['redborder']['services_group']['historical'] = %w(druid-historical) default['redborder']['services_group']['middlemanager'] = %w(druid-middlemanager) default['redborder']['services_group']['broker'] = %w(druid-broker) default['redborder']['services_group']['http2k'] = %w(http2k) -default['redborder']['services_group']['samza'] = %w(hadoop-nodemanager) default['redborder']['services_group']['webui'] = %w(nginx webui) default['redborder']['services_group']['f2k'] = %w(f2k) default['redborder']['services_group']['s3'] = %w(nginx s3) @@ -157,8 +148,6 @@ default['redborder']['services']['secor'] = false default['redborder']['services']['s3'] = false default['redborder']['services']['mongodb'] = false -default['redborder']['services']['hadoop-nodemanager'] = false -default['redborder']['services']['hadoop-resourcemanager'] = false default['redborder']['services']['redborder-monitor'] = true default['redborder']['services']['redborder-scanner'] = false default['redborder']['services']['snmp'] = true diff --git a/resources/metadata.rb b/resources/metadata.rb index 4eab1e86..d9665777 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -15,8 +15,6 @@ depends 'http2k' depends 'memcached' depends 'consul' -depends 'hadoop' -depends 'samza' depends 'nginx' depends 'geoip' depends 'snmp' diff --git a/resources/recipes/configure.rb b/resources/recipes/configure.rb index b4e4004b..a568ae0c 100644 --- a/resources/recipes/configure.rb +++ b/resources/recipes/configure.rb @@ -12,22 +12,15 @@ virtual_ips = node.run_state['virtual_ips'] virtual_ips_per_ip = node.run_state['virtual_ips_per_ip'] -begin - split_traffic_logstash_db = data_bag_item('rBglobal', 'splittraffic') - split_traffic_logstash = split_traffic_logstash_db['logstash'] -rescue - split_traffic_logstash = false -end - # bash 'upload_cookbooks' do # code 'bash /usr/lib/redborder/bin/rb_upload_cookbooks.sh' # only_if { ::File.exist?('/root/.upload-cookbooks') } # notifies :delete, 'file[/root/.upload-cookbooks]', :immediately # end -file '/root/.upload-cookbooks' do - action :nothing -end +# file '/root/.upload-cookbooks' do +# action :nothing +# end rb_common_config 'Configure common' do action :configure @@ -42,13 +35,14 @@ end consul_config 'Configure Consul Server' do - confdir node['consul']['confdir'] - datadir node['consul']['datadir'] - ipaddress node['ipaddress_sync'] cdomain node['redborder']['cdomain'] dns_local_ip node['consul']['dns_local_ip'] - (manager_services['consul'] ? (is_server true) : (is_server false)) + if manager_services['consul'] || manager_services['consul-client'] + confdir node['consul']['confdir'] + datadir node['consul']['datadir'] + ipaddress node['ipaddress_sync'] + (manager_services['consul'] ? (is_server true) : (is_server false)) action :add else action :remove @@ -56,37 +50,41 @@ end chef_server_config 'Configure chef services' do - memory node['redborder']['memory_services']['chef-server']['memory'] - postgresql false - postgresql_memory node['redborder']['memory_services']['postgresql']['memory'] - chef_active manager_services['chef-server'] - ipaddress node['ipaddress_sync'] if manager_services['chef-server'] + memory node['redborder']['memory_services']['chef-server']['memory'] + postgresql false + postgresql_memory node['redborder']['memory_services']['postgresql']['memory'] + chef_active manager_services['chef-server'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] end end -begin - vrrp_secrets = data_bag_item('passwords', 'vrrp') -rescue - vrrp_secrets = {} +vrrp_secrets = {} + +if manager_services['keepalived'] + begin + vrrp_secrets = data_bag_item('passwords', 'vrrp') + rescue + vrrp_secrets = {} + end end keepalived_config 'Configure keepalived' do - vrrp_secrets vrrp_secrets - virtual_ips virtual_ips - virtual_ips_per_ip virtual_ips_per_ip - managers managers - balanced_services node['redborder']['manager']['balanced'] - has_any_virtual_ip node.run_state['has_any_virtual_ip'] - manager_services manager_services - ipmgt node['ipaddress'] - iface_management node['redborder']['management_interface'] - ipaddress_sync node['ipaddress_sync'] - managers_per_service node['redborder']['managers_per_services'] if manager_services['keepalived'] + vrrp_secrets vrrp_secrets + virtual_ips virtual_ips + virtual_ips_per_ip virtual_ips_per_ip + managers managers + balanced_services node['redborder']['manager']['balanced'] + has_any_virtual_ip node.run_state['has_any_virtual_ip'] + manager_services manager_services + ipmgt node['ipaddress'] + iface_management node['redborder']['management_interface'] + ipaddress_sync node['ipaddress_sync'] + managers_per_service node['redborder']['managers_per_services'] action :add else action :remove @@ -94,11 +92,11 @@ end zookeeper_config 'Configure Zookeeper' do - port node['zookeeper']['port'] - memory node['redborder']['memory_services']['zookeeper']['memory'] - hosts node['redborder']['managers_per_services']['zookeeper'] - ipaddress node['ipaddress_sync'] if manager_services['zookeeper'] + port node['zookeeper']['port'] + memory node['redborder']['memory_services']['zookeeper']['memory'] + hosts node['redborder']['managers_per_services']['zookeeper'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] @@ -106,13 +104,13 @@ end kafka_config 'Configure Kafka' do - memory node['redborder']['memory_services']['kafka']['memory'] - maxsize node['redborder']['manager']['hd_services_current']['kafka'] - managers_list node['redborder']['managers_per_services']['kafka'] - zk_hosts node['redborder']['zookeeper']['zk_hosts'] - host_index node['redborder']['kafka']['host_index'] - ipaddress node['ipaddress_sync'] if manager_services['kafka'] + memory node['redborder']['memory_services']['kafka']['memory'] + maxsize node['redborder']['manager']['hd_services_current']['kafka'] + managers_list node['redborder']['managers_per_services']['kafka'] + zk_hosts node['redborder']['zookeeper']['zk_hosts'] + host_index node['redborder']['kafka']['host_index'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] @@ -149,10 +147,10 @@ end druid_coordinator 'Configure Druid Coordinator' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - memory_kb node['redborder']['memory_services']['druid-coordinator']['memory'] if manager_services['druid-coordinator'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-coordinator']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -160,10 +158,10 @@ end druid_overlord 'Configure Druid Overlord' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - memory_kb node['redborder']['memory_services']['druid-overlord']['memory'] if manager_services['druid-overlord'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-overlord']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -171,10 +169,10 @@ end druid_broker 'Configure Druid Broker' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - memory_kb node['redborder']['memory_services']['druid-broker']['memory'] if manager_services['druid-broker'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-broker']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -182,10 +180,10 @@ end druid_middlemanager 'Configure Druid MiddleManager' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - memory_kb node['redborder']['memory_services']['druid-middlemanager']['memory'] if manager_services['druid-middlemanager'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-middlemanager']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -193,10 +191,10 @@ end druid_historical 'Configure Druid Historical' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - memory_kb node['redborder']['memory_services']['druid-historical']['memory'] if manager_services['druid-historical'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + memory_kb node['redborder']['memory_services']['druid-historical']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -204,12 +202,12 @@ end druid_realtime 'Configure Druid Realtime' do - name node['hostname'] - ipaddress node['ipaddress_sync'] - zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] - partition_num node['redborder']['druid']['realtime']['partition_num'] - memory_kb node['redborder']['memory_services']['druid-realtime']['memory'] if manager_services['druid-realtime'] + name node['hostname'] + ipaddress node['ipaddress_sync'] + zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] + partition_num node['redborder']['druid']['realtime']['partition_num'] + memory_kb node['redborder']['memory_services']['druid-realtime']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -217,82 +215,38 @@ end memcached_config 'Configure Memcached' do - memory node['redborder']['memory_services']['memcached']['memory'] - ipaddress node['ipaddress_sync'] if manager_services['memcached'] + memory node['redborder']['memory_services']['memcached']['memory'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] end end -is_mongo_configured_consul = shell_out("curl -s http://localhost:8500/v1/health/service/mongodb | jq -r '.[].Checks[0].Status' | grep -q 'passing'") -get_consul_registered_ip = shell_out("curl -s http://localhost:8500/v1/health/service/mongodb | jq -r '.[].Service.Address' | head -n 1") - -mongodb_config 'Configure Mongodb' do - if manager_services['mongodb'] - if is_mongo_configured_consul.exitstatus == 0 - if node['ipaddress_sync'] == get_consul_registered_ip.stdout.strip - action [:add, :register] - else - action [:remove, :deregister] - end - else - action [:add, :register] - end - else - action [:remove, :deregister] - end -end - -hadoop_common 'Configure hadoop common resources' do - name node['hostname'] - zookeeper_hosts node['redborder']['zookeeper']['zk_hosts'] - memory_kb node['redborder']['memory_services']['hadoop-nodemanager']['memory'] - containersMemory node['redborder']['hadoop']['containersMemory'] - if manager_services['hadoop-nodemanager'] || manager_services['hadoop-resourcemanager'] - action :add - else - action :remove - end +enable_mongodb = false +if manager_services['mongodb'] + is_mongo_configured_consul = shell_out("curl -s http://localhost:8500/v1/health/service/mongodb | jq -r '.[].Checks[0].Status' | grep -q 'passing'") + get_consul_registered_ip = shell_out("curl -s http://localhost:8500/v1/health/service/mongodb | jq -r '.[].Service.Address' | head -n 1") + enable_mongodb = (is_mongo_configured_consul.exitstatus != 0) ? true : (node['ipaddress_sync'] == get_consul_registered_ip.stdout.strip) end -hadoop_resourcemanager 'Configure Hadoop ResourceManager' do - memory_kb node['redborder']['memory_services']['hadoop-resourcemanager']['memory'] - if manager_services['hadoop-resourcemanager'] - action [:add, :register] - else - action [:remove, :deregister] - end -end - -hadoop_nodemanager 'Configure Hadoop NodeManager' do - memory_kb node['redborder']['memory_services']['hadoop-nodemanager']['memory'] - if manager_services['hadoop-nodemanager'] +mongodb_config 'Configure Mongodb' do + if enable_mongodb action [:add, :register] else action [:remove, :deregister] end end -samza_config 'Configure samza applications' do - memory_per_container node['redborder']['samza']['memory_per_container'] - num_containers node['redborder']['samza']['num_containers'] - if manager_services['hadoop-nodemanager'] - action :add - else - action :remove - end -end - geoip_config 'Configure GeoIP' do action :add end snmp_config 'Configure snmp' do - hostname node['hostname'] - cdomain node['redborder']['cdomain'] if manager_services['snmp'] + hostname node['hostname'] + cdomain node['redborder']['cdomain'] action :add else action :remove @@ -300,13 +254,13 @@ end rbmonitor_config 'Configure redborder-monitor' do - name node['hostname'] - device_nodes node.run_state['sensors_info_all']['device-sensor'] - flow_nodes node.run_state['sensors_info_all']['flow-sensor'] - managers node['redborder']['managers_list'] - cluster node['redborder']['cluster_info'] - hostip node['redborder']['cluster_info'][name]['ip'] if manager_services['redborder-monitor'] + name node['hostname'] + device_nodes node.run_state['sensors_info_all']['device-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + managers node['redborder']['managers_list'] + cluster node['redborder']['cluster_info'] + hostip node['redborder']['cluster_info'][name]['ip'] action :add else action :remove @@ -314,8 +268,8 @@ end rbscanner_config 'Configure redborder-scanner' do - scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] if manager_services['redborder-scanner'] + scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] action [:add, :register] else action [:remove, :deregister] @@ -323,8 +277,8 @@ end nginx_config 'Configure Nginx' do - cdomain node['redborder']['cdomain'] if manager_services['nginx'] + cdomain node['redborder']['cdomain'] action [:add, :register] else action [:remove, :deregister] @@ -332,9 +286,9 @@ end nginx_config 'Configure Nginx Chef' do - service_name 'erchef' - cdomain node['redborder']['cdomain'] if manager_services['nginx'] && manager_services['chef-server'] + service_name 'erchef' + cdomain node['redborder']['cdomain'] action [:configure_certs, :add_erchef] else action :nothing @@ -342,9 +296,9 @@ end nginx_config 'Configure Nginx aioutliers' do - service_name 'rb-aioutliers' - cdomain node['redborder']['cdomain'] if manager_services['nginx'] && manager_services['rb-aioutliers'] + service_name 'rb-aioutliers' + cdomain node['redborder']['cdomain'] action [:configure_certs, :add_aioutliers] else action :nothing @@ -352,12 +306,12 @@ end webui_config 'Configure WebUI' do - hostname node['hostname'] - memcached_servers node['redborder']['managers_per_services']['memcached'] - memory_kb node['redborder']['memory_services']['webui']['memory'] - cdomain node['redborder']['cdomain'] - port node['redborder']['webui']['port'] if manager_services['webui'] + hostname node['hostname'] + memcached_servers node['redborder']['managers_per_services']['memcached'] + memory_kb node['redborder']['memory_services']['webui']['memory'] + cdomain node['redborder']['cdomain'] + port node['redborder']['webui']['port'] action [:add, :register, :configure_rsa] else action [:remove, :deregister] @@ -365,10 +319,10 @@ end webui_config 'Configure Nginx WebUI' do - hosts node['redborder']['webui']['hosts'] - cdomain node['redborder']['cdomain'] - port node['redborder']['webui']['port'] if manager_services['webui'] && manager_services['nginx'] + hosts node['redborder']['webui']['hosts'] + cdomain node['redborder']['cdomain'] + port node['redborder']['webui']['port'] action [:configure_certs, :add_webui_conf_nginx] else action :nothing @@ -376,17 +330,17 @@ end http2k_config 'Configure Http2k' do - domain node['redborder']['cdomain'] - kafka_hosts node['redborder']['managers_per_services']['kafka'] - memory node['redborder']['memory_services']['http2k']['memory'] - port node['redborder']['http2k']['port'] - proxy_nodes node.run_state['sensors_info']['proxy-sensor'] - ips_nodes node.run_state['sensors_info']['ips-sensor'] - ipsg_nodes node.run_state['sensors_info']['ipsg-sensor'] - ipscp_nodes node.run_state['sensors_info']['ipscp-sensor'] - organizations node['redborder']['organizations'] - locations_list node['redborder']['locations'] if manager_services['http2k'] + domain node['redborder']['cdomain'] + kafka_hosts node['redborder']['managers_per_services']['kafka'] + memory node['redborder']['memory_services']['http2k']['memory'] + port node['redborder']['http2k']['port'] + proxy_nodes node.run_state['sensors_info']['proxy-sensor'] + ips_nodes node.run_state['sensors_info']['ips-sensor'] + ipsg_nodes node.run_state['sensors_info']['ipsg-sensor'] + ipscp_nodes node.run_state['sensors_info']['ipscp-sensor'] + organizations node.run_state['organizations'] + locations_list node['redborder']['locations'] action [:add, :register] else action [:remove, :deregister] @@ -394,9 +348,9 @@ end http2k_config 'Configure Nginx Http2k' do - domain node['redborder']['cdomain'] - port node['redborder']['http2k']['port'] if manager_services['http2k'] && manager_services['nginx'] + domain node['redborder']['cdomain'] + port node['redborder']['http2k']['port'] action [:configure_certs, :add_http2k_conf_nginx] else action :nothing @@ -404,8 +358,8 @@ end f2k_config 'Configure f2k' do - sensors node.run_state['sensors_info']['flow-sensor'] if manager_services['f2k'] + sensors node.run_state['sensors_info']['flow-sensor'] action [:add, :register] else action [:remove, :deregister] @@ -413,27 +367,38 @@ end pmacct_config 'Configure pmacct' do - sensors node.run_state['sensors_info']['flow-sensor'] - kafka_hosts node['redborder']['managers_per_services']['kafka'] if manager_services['pmacct'] + sensors node.run_state['sensors_info']['flow-sensor'] + kafka_hosts node['redborder']['managers_per_services']['kafka'] action [:add, :register] else action [:remove, :deregister] end end +# Configure logstash +split_traffic = false + +if manager_services['logstash'] + begin + split_traffic = data_bag_item('rBglobal', 'splittraffic')['logstash'] + rescue + split_traffic = false + end +end + logstash_config 'Configure logstash' do - cdomain node['redborder']['cdomain'] - flow_nodes node.run_state['all_flow_sensors_info']['flow-sensor'] - namespaces node.run_state['namespaces'] - vault_nodes node.run_state['sensors_info_all']['vault-sensor'] - proxy_nodes node.run_state['sensors_info_all']['proxy-sensor'] - scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] - device_nodes node.run_state['sensors_info_all']['device-sensor'] - incidents_priority_filter node['redborder']['incidents_priority_filter'] - logstash_pipelines node.default['pipelines'] - split_traffic_logstash split_traffic_logstash - if !logstash_pipelines.nil? && !logstash_pipelines.empty? + if manager_services['logstash'] && node.run_state['pipelines'] && !node.run_state['pipelines'].empty? + cdomain node['redborder']['cdomain'] + flow_nodes node.run_state['flow_sensors_info'] + namespaces node.run_state['namespaces'] + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + proxy_nodes node.run_state['sensors_info_all']['proxy-sensor'] + scanner_nodes node.run_state['sensors_info_all']['scanner-sensor'] + device_nodes node.run_state['sensors_info_all']['device-sensor'] + incidents_priority_filter node['redborder']['incidents_priority_filter'] + logstash_pipelines node.run_state['pipelines'] + split_traffic_logstash split_traffic action [:add, :register] else action [:remove, :deregister] @@ -441,8 +406,8 @@ end rbdswatcher_config 'Configure redborder-dswatcher' do - cdomain node['redborder']['cdomain'] if manager_services['redborder-dswatcher'] + cdomain node['redborder']['cdomain'] action [:add, :register] else action [:remove, :deregister] @@ -450,8 +415,8 @@ end rbevents_counter_config 'Configure redborder-events-counter' do - cdomain node['redborder']['cdomain'] if manager_services['redborder-events-counter'] + cdomain node['redborder']['cdomain'] action [:add, :register] else action [:remove, :deregister] @@ -459,9 +424,9 @@ end rsyslog_config 'Configure rsyslog' do - vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + node.run_state['sensors_info_all']['cep-sensor'] - ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] if manager_services['rsyslog'] + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + node.run_state['sensors_info_all']['cep-sensor'] + ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] action [:add, :register] else action [:remove, :deregister] @@ -469,11 +434,11 @@ end rbnmsp_config 'Configure redborder-nmsp' do - memory node['redborder']['memory_services']['redborder-nmsp']['memory'] - proxy_nodes node.run_state['sensors_info_all']['proxy-sensor'] - flow_nodes node.run_state['sensors_info_all']['flow-sensor'] - hosts node['redborder']['zookeeper']['zk_hosts'] if manager_services['redborder-nmsp'] + memory node['redborder']['memory_services']['redborder-nmsp']['memory'] + proxy_nodes node.run_state['sensors_info_all']['proxy-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + hosts node['redborder']['zookeeper']['zk_hosts'] action [:add, :configure_keys, :register] else action [:remove, :deregister] @@ -481,11 +446,11 @@ end n2klocd_config 'Configure n2klocd' do - mse_nodes node.run_state['sensors_info_all']['mse-sensor'] - meraki_nodes node.run_state['sensors_info_all']['meraki-sensor'] - n2klocd_managers node['redborder']['managers_per_services']['n2klocd'] - memory node['redborder']['memory_services']['n2klocd']['memory'] if manager_services['n2klocd'] + mse_nodes node.run_state['sensors_info_all']['mse-sensor'] + meraki_nodes node.run_state['sensors_info_all']['meraki-sensor'] + n2klocd_managers node['redborder']['managers_per_services']['n2klocd'] + memory node['redborder']['memory_services']['n2klocd']['memory'] action [:add, :register] else action [:remove, :deregister] @@ -493,9 +458,8 @@ end rbale_config 'Configure redborder-ale' do - ale_sensors = node.run_state['sensors_info_all']['ale-sensor'] - if !ale_sensors.empty? - ale_nodes ale_sensors + if manager_services['redborder-ale'] + ale_nodes node.run_state['sensors_info_all']['ale-sensor'] action [:add, :register] else action [:remove, :deregister] @@ -503,7 +467,7 @@ end rblogstatter_config 'Configure redborder-logstatter' do - if node['redborder']['services']['rb-logstatter'] + if manager_services['rb-logstatter'] action :add else action :remove @@ -511,9 +475,9 @@ end rb_arubacentral_config 'Configure rb-arubacentral' do - arubacentral_nodes node.run_state['sensors_info_all']['arubacentral-sensor'] - flow_nodes node.run_state['sensors_info_all']['flow-sensor'] - if node['redborder']['services']['rb-arubacentral'] + if manager_services['rb-arubacentral'] + arubacentral_nodes node.run_state['sensors_info_all']['arubacentral-sensor'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] action :add else action :remove @@ -522,7 +486,7 @@ # freeradius_config 'Configure radiusd' do # flow_nodes node.run_state['sensors_info_all']['flow-sensor'] -# action (node['redborder']['services']['radiusd'] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) +# action (manager_services['radiusd'] ? [:config_common, :config_manager, :register] : [:remove, :deregister]) # end rbaioutliers_config 'Configure rb-aioutliers' do @@ -534,10 +498,10 @@ end rbcep_config 'Configure redborder-cep' do - flow_nodes node.run_state['sensors_info_all']['flow-sensor'] - vault_nodes node.run_state['sensors_info_all']['vault-sensor'] - ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] - if node['redborder']['services']['redborder-cep'] + if manager_services['redborder-cep'] + flow_nodes node.run_state['sensors_info_all']['flow-sensor'] + vault_nodes node.run_state['sensors_info_all']['vault-sensor'] + ips_nodes node.run_state['sensors_info_all']['ips-sensor'] + node.run_state['sensors_info_all']['ipsv2-sensor'] + node.run_state['sensors_info_all']['ipscp-sensor'] action [:add, :register] else action [:remove, :deregister] @@ -545,10 +509,10 @@ end mem2incident_config 'Configure redborder-mem2incident' do - cdomain node['redborder']['cdomain'] - memcached_servers node['redborder']['managers_per_services']['memcached'].map { |s| "#{s}:#{node['redborder']['memcached']['port']}" } - auth_token node.run_state['auth_token'] if manager_services['redborder-mem2incident'] + cdomain node['redborder']['cdomain'] + memcached_servers node['redborder']['managers_per_services']['memcached'].map { |s| "#{s}:#{node['redborder']['memcached']['port']}" } + auth_token node.run_state['auth_token'] action [:add, :register] else action [:remove, :deregister] @@ -556,10 +520,10 @@ end rb_ai_config 'Configure redborder-ai' do - ai_selected_model node['redborder']['ai_selected_model'] - cpus node['redborder']['redborder-ai']['cpus'] - ipaddress node['ipaddress_sync'] if manager_services['redborder-ai'] + ai_selected_model node['redborder']['ai_selected_model'] + cpus node['redborder']['redborder-ai']['cpus'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] @@ -567,7 +531,7 @@ end rb_postfix_config 'Configure postfix' do - if node['redborder']['services']['postfix'] + if manager_services['postfix'] action :add else action :remove @@ -575,16 +539,21 @@ end rbcgroup_config 'Configure cgroups' do + check_cgroups node.run_state['cluster_installed'] action :add end rb_clamav_config 'Configure ClamAV' do - action(manager_services['clamav'] ? :add : :remove) + if manager_services['clamav'] + action :add + else + action :remove + end end rb_chrony_config 'Configure Chrony' do - ntp_servers node['redborder']['ntp']['servers'] if manager_services['chrony'] + ntp_servers node['redborder']['ntp']['servers'] action :add else action :remove @@ -599,28 +568,33 @@ end postgresql_config 'Configure postgresql' do - cdomain node['redborder']['cdomain'] - ipaddress node['ipaddress_sync'] if manager_services['postgresql'] && external_services['postgresql'] == 'onpremise' + cdomain node['redborder']['cdomain'] + ipaddress node['ipaddress_sync'] action [:add, :register] else action [:remove, :deregister] end end -begin - s3_secrets = data_bag_item('passwords', 's3') -rescue - ssh_secrets = {} +s3_secrets = {} + +if manager_services['s3'] && (external_services['s3'] == 'onpremise') + begin + s3_secrets = data_bag_item('passwords', 's3') + rescue + s3_secrets = {} + end end # Allow only s3 onpremise nodes for now.. minio_config 'Configure S3 (minio)' do managers_with_minio node['redborder']['managers_per_services']['s3'] - ipaddress node['ipaddress_sync'] access_key_id s3_secrets['s3_access_key_id'] secret_key_id s3_secrets['s3_secret_key_id'] if manager_services['s3'] && (external_services['s3'] == 'onpremise') + ipaddress node['ipaddress_sync'] + action [:add, :register] action [:add, :register, :add_mcli] else action [:remove, :deregister, :add_mcli] @@ -628,24 +602,28 @@ end # First configure the cert for the service before configuring nginx -if manager_services['s3'] - nginx_config 'Configure S3 certs' do +nginx_config 'Configure S3 certs' do + if manager_services['s3'] service_name 's3' cdomain node['redborder']['cdomain'] action :configure_certs + else + action :nothing end end # Configure Nginx s3 onpremise nodes for now.. minio_config 'Configure Nginx S3 (minio)' do - s3_hosts node['redborder']['s3']['s3_hosts'] if manager_services['s3'] && (external_services['s3'] == 'onpremise') + s3_hosts node['redborder']['s3']['s3_hosts'] action [:add_s3_conf_nginx] else action :nothing end end +ssh_secrets = {} + begin ssh_secrets = data_bag_item('passwords', 'ssh') rescue @@ -699,7 +677,7 @@ end # MOTD -cluster_info = node['redborder']['cluster_info'] +cluster_uuid_db = {} begin cluster_uuid_db = data_bag_item('rBglobal', 'cluster') @@ -707,8 +685,6 @@ cluster_uuid_db = {} end -cluster_installed = File.exist?('/etc/redborder/cluster-installed.txt') - template '/etc/motd' do source 'motd.erb' owner 'root' @@ -716,8 +692,8 @@ mode '0644' retries 2 backup false - variables(cluster_info: cluster_info, + variables(cluster_info: node['redborder']['cluster_info'], uuid: cluster_uuid_db['uuid'], manager_services: manager_services, - cluster_finished: cluster_installed) + cluster_finished: node.run_state['cluster_installed']) end diff --git a/resources/recipes/configure_cron_tasks.rb b/resources/recipes/configure_cron_tasks.rb index 47eb75b3..4caf714d 100644 --- a/resources/recipes/configure_cron_tasks.rb +++ b/resources/recipes/configure_cron_tasks.rb @@ -6,11 +6,9 @@ # Services configuration manager_services = manager_services() -cluster_installed = File.exist?('/etc/redborder/cluster-installed.txt') - # Druid cron_d 'clean_segments_daily' do - if (manager_services['druid-coordinator'] || manager_services['druid-historical']) && cluster_installed + if (manager_services['druid-coordinator'] || manager_services['druid-historical']) && node.run_state['cluster_installed'] action :create else action :delete diff --git a/resources/recipes/prepare_system.rb b/resources/recipes/prepare_system.rb index 1d6d4c71..057a80b8 100644 --- a/resources/recipes/prepare_system.rb +++ b/resources/recipes/prepare_system.rb @@ -11,12 +11,11 @@ end # Set services_group related with the node mode (core, full, ...) -mode = node['redborder']['mode'] -node['redborder']['services_group'][mode].each { |s| node.default['redborder']['services'][s] = true } +node['redborder']['services_group'][node['redborder']['mode']].each { |s| node.default['redborder']['services'][s] = true } -if mode != 'core' || mode != 'full' - node.default['redborder']['services']['consul-client'] = true -end +node.default['redborder']['services']['consul-client'] = (node['redborder']['mode'] != 'core' && node['redborder']['mode'] != 'full') + +node.run_state['cluster_installed'] = File.exist?('/etc/redborder/cluster-installed.txt') # Set :ipaddress_sync ipaddress_sync = node['ipaddress'] @@ -54,28 +53,33 @@ end service 'chef-client' do - if node['redborder']['services']['chef-client'] + if node['redborder']['services']['chef-client'] && node.run_state['cluster_installed'] action [:enable, :start] else action [:stop] end end -# get managers information(name, ip, services...) -cdomain = '' -File.open('/etc/redborder/cdomain') { |f| cdomain = f.readline.chomp } -node.default['redborder']['cdomain'] = cdomain +# get cluster domain +node.default['redborder']['cdomain'] = File.read('/etc/redborder/cdomain').chomp # get managers information(name, ip, services...) node.default['redborder']['cluster_info'] = get_cluster_info # manager services -node.run_state['manager_services'] = manager_services() +node.run_state['manager_services'] = manager_services node.default['redborder']['manager']['services']['current'] = node.run_state['manager_services'] # get managers sorted by service node.default['redborder']['managers_per_services'] = managers_per_service +node.run_state['organizations'] = get_orgs if node['redborder']['services']['http2k'] + +if node['redborder']['services']['logstash'] + node.run_state['pipelines'] = get_pipelines + node.run_state['flow_sensors_info'] = get_all_flow_sensors_info['flow-sensor'] +end + # get elasticache nodes begin elasticache = data_bag_item('rBglobal', 'elasticache') @@ -96,20 +100,11 @@ node.default['redborder']['memcached']['hosts'] = memcached_hosts end -# get organizations for http2k -node.default['redborder']['organizations'] = get_orgs() if node['redborder']['services']['http2k'] - # get sensors info -node.run_state['sensors_info'] = get_sensors_info() +node.run_state['sensors_info'] = get_sensors_info # get sensors info full info -node.run_state['sensors_info_all'] = get_sensors_all_info() - -# get sensors info of all flow sensors -node.run_state['all_flow_sensors_info'] = get_all_flow_sensors_info() - -# get logstash pipelines -node.default['pipelines'] = get_pipelines() +node.run_state['sensors_info_all'] = get_sensors_all_info # get namespaces node.run_state['namespaces'] = get_namespaces @@ -133,7 +128,7 @@ if File.exist?'/etc/lock/keepalived' node.run_state['manager_services']['keepalived'] = false elsif node['redborder'].nil? || node['redborder']['dmidecode'].nil? || node['redborder']['dmidecode']['manufacturer'].nil? || node['redborder']['dmidecode']['manufacturer'].to_s.downcase == 'xen' - if manager_index > 0 && !cluster_installed + if manager_index > 0 && !node.run_state['cluster_installed'] node.run_state['manager_services']['keepalived'] = false else node.run_state['manager_services']['keepalived'] = node.run_state['has_any_virtual_ip'] and !File.exist?'/etc/lock/keepalived' diff --git a/resources/templates/default/motd.erb b/resources/templates/default/motd.erb index 58d805fd..60c85e33 100644 --- a/resources/templates/default/motd.erb +++ b/resources/templates/default/motd.erb @@ -41,12 +41,12 @@ end if node["redborder"]["cluster"]["services"].select {|x| x["name"]==s}.first["ok"] - manager_services << s.gsub("druid-", "").gsub("hadoop_", "") if (s!="chef-client" and s!="keepalived" and s!="riak-cs" and s!="opscode-solr4" and s!="opscode-expander" and s!="rabbitmq" and s!="redborder-monitor" and s!="rb-discover") + manager_services << s.gsub("druid-", "") if (s!="chef-client" and s!="keepalived" and s!="riak-cs" and s!="opscode-solr4" and s!="opscode-expander" and s!="rabbitmq" and s!="redborder-monitor" and s!="rb-discover") else - manager_services2 << s.gsub("druid-", "").gsub("hadoop_", "") + manager_services2 << s.gsub("druid-", "") end rescue - manager_services << "#{s}?".gsub("druid-", "").gsub("hadoop_", "") + manager_services << "#{s}?".gsub("druid-", "") end end %> diff --git a/resources/templates/default/redborder-manager.erb b/resources/templates/default/redborder-manager.erb index 19ef307c..2a3b6898 100644 --- a/resources/templates/default/redborder-manager.erb +++ b/resources/templates/default/redborder-manager.erb @@ -2,7 +2,6 @@ Defaults:root !requiretty Defaults:redborder !requiretty Defaults:redborder-monitor !requiretty, !syslog Defaults:webui !requiretty -Defaults:hadoop !requiretty redborder ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_wakeup_chef.sh, /opt/rb/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef.sh redborder-monitor ALL= NOPASSWD: /usr/lib/redborder/bin/rb_get_sensor.sh, /usr/lib/redborder/bin/cachestat.sh, /bin/nice -n 19 /usr/sbin/fping -p 1 -c 10 *, /usr/lib/redborder/bin/rb_mem.sh * webui ALL= NOPASSWD:SETENV: /usr/lib/redborder/bin/rb_get_raw_vault.sh *, /usr/lib/redborder/bin/rb_update_geodb_cookbook.sh, /usr/lib/redborder/bin/rb_update_geoip, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -c, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef -n *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_wakeup_chef, /usr/lib/redborder/bin/rb_reassign_partitions.sh -d -e, /usr/lib/redborder/bin/rb_pcp.sh up *, /usr/lib/redborder/bin/rb_pcp.sh recovery *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_pcp.sh status, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_get_zkinfo.sh, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_create_bulkstats_columns.sh *, /bin/env BOOTUP=none /usr/lib/redborder/bin/rb_create_rsa.sh *