Merge pull request #8 from codebude/master

Added metrics for service health status

code/exporter.py

@@ -26,10 +26,13 @@
     SCRAPE_INTERVAL = 30
 
 
-HETZNER_CLOUD_API_URL = 'https://api.hetzner.cloud/v1/load_balancers/'
+HETZNER_CLOUD_API_URL_BASE = 'https://api.hetzner.cloud/v1'
+HETZNER_CLOUD_API_URL_LB = f'{HETZNER_CLOUD_API_URL_BASE}/load_balancers/'
+HETZNER_CLOUD_API_URL_SERVER = f'{HETZNER_CLOUD_API_URL_BASE}/servers/'
 
-def get_all_load_balancers_ids():
-    url = f'{HETZNER_CLOUD_API_URL}'
+
+def get_all_load_balancers_ids() -> dict:
+    url = f'{HETZNER_CLOUD_API_URL_LB}'
     headers = {
         'Content-type': "application/json",
         'Authorization': f"Bearer {access_token}"
@@ -39,8 +42,8 @@ def get_all_load_balancers_ids():
     return get.json()['load_balancers']
 
 
-def get_load_balancer_info(lbid):
-    url = f'{HETZNER_CLOUD_API_URL}{lbid}'
+def get_load_balancer_info(lbid) -> dict:
+    url = f'{HETZNER_CLOUD_API_URL_LB}{lbid}'
 
     headers = {
         'Content-type': "application/json",
@@ -51,12 +54,51 @@ def get_load_balancer_info(lbid):
     return get.json()
 
 
+def get_all_server_names() -> dict:
+    url = f'{HETZNER_CLOUD_API_URL_SERVER}'
+
+    headers = {
+        'Content-type': "application/json",
+        'Authorization': f"Bearer {access_token}"
+    }
+
+    get = requests.get(url, headers=headers)
+    return {x['id']: x['name'] for x in get.json()['servers']}
+
+
+def get_server_info(server_id) -> dict:
+    url = f'{HETZNER_CLOUD_API_URL_SERVER}{server_id}'
+
+    headers = {
+        'Content-type': "application/json",
+        'Authorization': f"Bearer {access_token}"
+    }
+
+    get = requests.get(url, headers=headers)
+    return get.json()
+
+
+def get_server_name_from_cache(server_id: str) -> str:
+    global server_name_cache
+    if server_id in server_name_cache:
+        return server_name_cache[server_id]
+    else:
+        # Refresh cache
+        server_name_cache = get_all_server_names()
+        # Check again
+        if server_id in server_name_cache:
+            return server_name_cache[server_id]
+        else:
+            # If still not found, return id as name
+            return server_id
+
+
 def get_metrics(metrics_type, lbid):
     utc_offset_sec = time.altzone if time.localtime().tm_isdst else time.timezone
     utc_offset = datetime.timedelta(seconds=-utc_offset_sec)
     hetzner_date = datetime.datetime.now().replace(tzinfo=datetime.timezone(offset=utc_offset)).isoformat()
 
-    url = f"{HETZNER_CLOUD_API_URL}{lbid}/metrics"
+    url = f"{HETZNER_CLOUD_API_URL_LB}{lbid}/metrics"
 
     headers = {
         'Content-type': "application/json",
@@ -95,7 +137,7 @@ def get_metrics(metrics_type, lbid):
         try:
             load_balancer_name = load_balancer['name']
         except Exception as e:
-            print('Couldnt get field', e )
+            print("Couldn't get field", e )
             sys.exit(1)
 
 
@@ -111,18 +153,24 @@ def get_metrics(metrics_type, lbid):
 
     print(f'\nScrape intreval: {SCRAPE_INTERVAL} seconds')
 
+    print('\nBuilding server name cache from Hetzner for labeling ...')
+    server_name_cache = get_all_server_names()
+    print(f'Retrieved {len(server_name_cache.keys())} server names from Hetzner ...\n')
+
     id_name_list = ['hetzner_load_balancer_id', 'hetzner_load_balancer_name']
     hetzner_load_balancer_info = Info('hetzner_load_balancer', 'Hetzner Load Balancer Exporter build info')
     hetzner_openconnections = Gauge('hetzner_load_balancer_open_connections', 'Open Connections on Hetzner Load Balancer', id_name_list)
     hetzner_connections_per_second = Gauge('hetzner_load_balancer_connections_per_second', 'Connections per Second on Hetzner Load Balancer', id_name_list)
     hetzner_requests_per_second = Gauge('hetzner_load_balancer_requests_per_second', 'Requests per Second on Hetzner Load Balancer', id_name_list)
     hetzner_bandwidth_in = Gauge('hetzner_load_balancer_bandwidth_in', 'Bandwidth in on Hetzner Load Balancer', id_name_list)
     hetzner_bandwidth_out = Gauge('hetzner_load_balancer_bandwidth_out', 'Bandwidth out on Hetzner Load Balancer', id_name_list)
+    id_name_service_list = id_name_list + ['hetzner_target_id', 'hetzner_target_name', 'hetzner_target_port']
+    hetzner_service_state = Gauge('hetzner_load_balancer_service_state', 'Health status of Load Balancer\'s services', id_name_service_list)
 
     start_http_server(8000)
     print('\nHetzner Load Balancer Exporter started')
     print('Visit http://localhost:8000/ to view the metrics')
-    hetzner_load_balancer_info.info({'version': '2.0.0', 'buildhost': 'drake0103@gmail.com'})
+    hetzner_load_balancer_info.info({'version': '2.0.0', 'buildhost': 'netblognet@gmail.com'})
 
     while True:
         for load_balancer_id, lb_name, load_balancer_type in load_balancer_full_list:
@@ -137,4 +185,14 @@ def get_metrics(metrics_type, lbid):
                                 hetzner_load_balancer_name=lb_name).set(get_metrics('bandwidth',load_balancer_id)["metrics"]["time_series"]["bandwidth.in"]["values"][0][1])
             hetzner_bandwidth_out.labels(hetzner_load_balancer_id=load_balancer_id,
                                 hetzner_load_balancer_name=lb_name).set(get_metrics('bandwidth',load_balancer_id)["metrics"]["time_series"]["bandwidth.out"]["values"][0][1])
+
+            lb_info = get_load_balancer_info(load_balancer_id)['load_balancer']
+            for target in [x for x in lb_info['targets'] if x['type'] == 'server']:
+                for health_status in target['health_status']:
+                    hetzner_service_state.labels(hetzner_load_balancer_id=load_balancer_id,
+                                                 hetzner_load_balancer_name=lb_name,
+                                                 hetzner_target_id=target['server']['id'],
+                                                 hetzner_target_name=get_server_name_from_cache(target['server']['id']),
+                                                 hetzner_target_port=health_status['listen_port'])\
+                        .set((1 if health_status['status'] == 'healthy' else 0))
         time.sleep(int(SCRAPE_INTERVAL))