From aa3ceb91d1da78421a5b0f968eb31f433eda46e4 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 19 Mar 2021 21:00:54 +0100 Subject: [PATCH 01/10] Move population to the 1st row, with other node numbers --- puppetboard/templates/index.html | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index f22309958..923c59c28 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -13,7 +13,7 @@ {% endif %}
-
+

@@ -50,13 +50,20 @@

unreported in the last {{ config.UNRESPONSIVE_HOURS }} hours

+
+

+ {{ metrics['num_nodes'] }} + {% if metrics['num_nodes'] == 1 %} node {% else %} nodes {% endif %} +

+ total population +
-
+
+
+
-

{{metrics['num_nodes']}}

- Population

{{metrics['num_resources']}}

From 0308d238f949f5fd72a41c4477550ee7db575126 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 19 Mar 2021 21:15:13 +0100 Subject: [PATCH 02/10] Allow disabling 2 sections of the front page (Overview). Fixes #544 and #533. --- README.md | 2 ++ puppetboard/default_settings.py | 2 ++ puppetboard/docker_settings.py | 4 ++++ puppetboard/templates/index.html | 4 ++++ 4 files changed, 12 insertions(+) diff --git a/README.md b/README.md index f0322d14a..42cdd6ec4 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,8 @@ Other settings that might be interesting in no particular order: in the last report. Otherwise shows only 'some' string if there are resources with given status. Setting this to `False` gives performance benefits, especially in big Puppet environments (more than few hundreds of nodes). Defaults to `True`. +- `RESOURCES_STATS_ENABLED`: If set then such section of the front page (Overview) is shown. Defaults to `True`. +- `NODES_STATUS_DETAIL_ENABLED`: If set then such section of the front page (Overview) is shown. Defaults to `True`. - `DEV_LISTEN_HOST`: For use with dev.py for development. Default is localhost - `DEV_LISTEN_PORT`: For use with dev.py for development. Default is 5000 diff --git a/puppetboard/default_settings.py b/puppetboard/default_settings.py index a9d48b1f8..e8c533d9a 100644 --- a/puppetboard/default_settings.py +++ b/puppetboard/default_settings.py @@ -53,3 +53,5 @@ DAILY_REPORTS_CHART_ENABLED = True DAILY_REPORTS_CHART_DAYS = 8 WITH_EVENT_NUMBERS = True +RESOURCES_STATS_ENABLED = True +NODES_STATUS_DETAIL_ENABLED = True diff --git a/puppetboard/docker_settings.py b/puppetboard/docker_settings.py index 6175d6322..d1f6be127 100644 --- a/puppetboard/docker_settings.py +++ b/puppetboard/docker_settings.py @@ -95,3 +95,7 @@ def coerce_bool(v, default): DAILY_REPORTS_CHART_DAYS = int(os.getenv('DAILY_REPORTS_CHART_DAYS', '8')) WITH_EVENT_NUMBERS = coerce_bool(os.getenv('WITH_EVENT_NUMBERS'), True) + +RESOURCES_STATS_ENABLED = coerce_bool(os.getenv('RESOURCES_STATS_ENABLED'), True) + +NODES_STATUS_DETAIL_ENABLED = coerce_bool(os.getenv('NODES_STATUS_DETAIL_ENABLED'), True) diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index 923c59c28..b5ed6a262 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -58,6 +58,7 @@

total population

+ {% if config.RESOURCES_STATS_ENABLED %}
@@ -74,11 +75,13 @@

{{metrics['avg_resources_node']} Avg. resources/node

+ {% endif %} {% if config.DAILY_REPORTS_CHART_ENABLED %}
{% endif %} + {% if config.NODES_STATUS_DETAIL_ENABLED %}
@@ -132,5 +135,6 @@

Nodes status detail

{% endif %}
+ {% endif %}
{% endblock content %} From 9885f365dffa84ce5b6978742d6f82d0e350f961 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 19 Mar 2021 22:09:04 +0100 Subject: [PATCH 03/10] Organize docs for and cleanup the settings --- README.md | 102 ++++++++++++++++++++------------- docs/EL7.md | 1 - puppetboard/docker_settings.py | 2 +- 3 files changed, 62 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 42cdd6ec4..8f898bfd7 100644 --- a/README.md +++ b/README.md @@ -123,53 +123,73 @@ For information about how to generate the correct keys please refer to the [pypuppetdb documentation](https://pypuppetdb.readthedocs.io/en/latest/connecting.html#ssl). Alternatively it is possible to explicitly specify the protocol to be used setting the `PUPPETDB_PROTO` variable. -Other settings that might be interesting in no particular order: +Other settings that might be interesting: -- `SECRET_KEY`: Refer to [Flask documentation](https://flask.palletsprojects.com/en/1.1.x/quickstart/#sessions), - section "How to generate good secret keys" for more info. Defaults to a random 24-char string generated by - `os.random(24)`. -- `PUPPETDB_TIMEOUT`: Defaults to 20 seconds but you might need to increase this value. It depends on how big the - results are when querying PuppetDB. This behaviour will change in a future release when pagination will be introduced. -- `UNRESPONSIVE_HOURS`: The amount of hours since the last check-in after which a node is considered unresponsive. -- `LOGLEVEL`: A string representing the loglevel. It defaults to `'info'` but can be changed to `'warning'` or - `'critical'` for less verbose logging or `'debug'` for more information. -- `ENABLE_QUERY`: Defaults to `True` causing a Query tab to show up in the web interface allowing users to write - and execute arbitrary queries against a set of endpoints in PuppetDB. Change this to `False` to disable this. - See `ENABLED_QUERY_ENDPOINTS` to fine-tune which endpoints are allowed. -- `ENABLED_QUERY_ENDPOINTS`: If `ENABLE_QUERY` is `True`, allow to fine tune the endpoints of PuppetDB APIs that - can be queried. It must be a list of strings of PuppetDB endpoints for which the query is enabled. - See the `QUERY_ENDPOINTS` constant in the `puppetboard.app` module for a list of the available endpoints. -- `GRAPH_TYPE`: Specify the type of graph to display. Default is - pie, other good option is donut. Other choices can be found here: - \_C3JS\_documentation\` -- `GRAPH_FACTS`: A list of fact names to tell PuppetBoard to generate a pie-chart on the fact page. With some fact - values being unique per node, like ipaddress, uuid, and serial number, as well as structured facts it was no longer - feasible to generate a graph for everything. -- `INVENTORY_FACTS`: A list of tuples that serve as the column header and the fact name to search for to create +#### General UI + +* `LOCALISE_TIMESTAMP`: Make the timestamps use the timezone provided by your browser. Defaults to `True`. +* `REFRESH_RATE`: The number of seconds between automatic page refreshes. Defaults to `30`. Set to `0` to disable. +* `DEFAULT_ENVIRONMENT`: The default value of the Puppet environment filter. Defaults to `'production'`. Set to `'*'` + to show all environments. +* `INVENTORY_FACTS`: A list of tuples that serve as the column header and the fact name to search for to create the inventory page. If a fact is not found for a node then `undef` is printed. -- `ENABLE_CATALOG`: If set to `True` allows the user to view a node's latest catalog. This includes all managed +* `ENABLE_QUERY`: If enabled then the Query tab is available which allows users to send arbitrary queries against + a set of endpoints in PuppetDB. Defaults to `True`. +* `ENABLED_QUERY_ENDPOINTS`: If `ENABLE_QUERY` is `True`, it allows restricting the list of the available endpoints. + It must be a list of strings of PuppetDB endpoints for which the query is enabled. The default is empty, which + means that all endpoints are allowed. See the `QUERY_ENDPOINTS` constant in `forms.py` for a list of the available + endpoints. +* `ENABLE_CATALOG`: If enabled then it allows the user to view a node's latest catalog. This includes all managed resources, their file-system locations and their relationships, if available. Defaults to `False`. -- `REFRESH_RATE`: Defaults to `30` the number of seconds to wait until the index page is automatically refreshed. -- `DEFAULT_ENVIRONMENT`: Defaults to `'production'`, as the name suggests, load all information filtered by this - environment value. -- `REPORTS_COUNT`: Defaults to `10` the limit of the number of reports to load on the node or any reports page. -- `OFFLINE_MODE`: If set to `True` load static assets (jquery, semantic-ui, etc) from the local web server instead - of a CDN. Defaults to `False`. -- `DAILY_REPORTS_CHART_ENABLED`: Enable the use of daily chart graphs when looking at dashboard and node view. -- `DAILY_REPORTS_CHART_DAYS`: Number of days to show history for on the daily report graphs. -- `DISPLAYED_METRICS`: Metrics to show when displaying node summary. Example: `'resources.total'`, `'events.noop'`. -- `TABLE_COUNT_SELECTOR`: Configure the dropdown to limit number of hosts to show per page. -- `LITTLE_TABLE_COUNT`: Default number of reports to show when when looking at a node. -- `NORMAL_TABLE_COUNT`: Default number of nodes to show when displaying reports and catalog nodes. -- `LOCALISE_TIMESTAMP`: Normalize time based on localserver time. -- `WITH_EVENT_NUMBERS`: If set to `True` then Overview and Nodes list shows exact number of changed resources + +#### Overview page + +* `RESOURCES_STATS_ENABLED`: If set then the section with "Resources managed" and "Avg. resources/node" + of the Overview page is shown. Defaults to `True`. +* `DAILY_REPORTS_CHART_ENABLED`: Enable the use of daily chart graphs when looking at Overview and Node view. + Defaults to `True`. +* `DAILY_REPORTS_CHART_DAYS`: Number of days to show history for on the daily report graphs. Defaults to `8`. +* `NODES_STATUS_DETAIL_ENABLED`: If set then such section of the Overview page is shown. Defaults to `True`. +* `UNRESPONSIVE_HOURS`: The amount of hours since the last check-in after which a node is considered unreported. + Defaults to `2`. + +#### Lists of nodes (Overview and Nodes pages) + +* `WITH_EVENT_NUMBERS`: If set to `True` then Overview and Nodes list shows exact number of changed resources in the last report. Otherwise shows only 'some' string if there are resources with given status. Setting this to `False` gives performance benefits, especially in big Puppet environments (more than few hundreds of nodes). Defaults to `True`. -- `RESOURCES_STATS_ENABLED`: If set then such section of the front page (Overview) is shown. Defaults to `True`. -- `NODES_STATUS_DETAIL_ENABLED`: If set then such section of the front page (Overview) is shown. Defaults to `True`. -- `DEV_LISTEN_HOST`: For use with dev.py for development. Default is localhost -- `DEV_LISTEN_PORT`: For use with dev.py for development. Default is 5000 +* `DISPLAYED_METRICS`: Metrics to show as the node "Status", to the right of the last report result. + Defaults to `['resources.total', 'events.failure', 'events.success', 'resources.skipped', 'events.noop']`. + +#### Facts page + +* `GRAPH_FACTS`: A list of fact names to tell Puppetboard to generate a pie-chart on the fact page. With some fact + values being unique per node, like ipaddress, uuid, and serial number, as well as structured facts it was no longer + feasible to generate a graph for everything. +* `GRAPH_TYPE`: Specify the type of graph to display. Default is pie, other good option is donut. Other choices + can be found here: [C3.js examples](https://c3js.org/examples.html) + +#### Various UI + +* `LITTLE_TABLE_COUNT`: Default number of reports to show when looking at a node. Defaults to `10`. +* `NORMAL_TABLE_COUNT`: Default number of nodes to show when displaying reports and catalog nodes. Defaults to `100`. +* `TABLE_COUNT_SELECTOR`: Configure the dropdown to limit number of hosts to show per page. + Defaults to `[10, 20, 50, 100, 500]`. + +#### Other + +* `PUPPETDB_TIMEOUT`: Defaults to 20 seconds but you might need to increase this value. It depends on how big the + results are when querying PuppetDB. This behaviour will change in a future release when pagination will be introduced. +* `LOGLEVEL`: A string representing the loglevel. It defaults to `'info'` but can be changed to `'warning'` or + `'critical'` for less verbose logging or `'debug'` for more information. +* `SECRET_KEY`: Refer to [Flask documentation](https://flask.palletsprojects.com/en/1.1.x/quickstart/#sessions), + section "How to generate good secret keys" for more info. Defaults to a random 24-char string generated by + `os.random(24)`. +* `OFFLINE_MODE`: If set to `True` load static assets (jquery, semantic-ui, etc) from the local web server instead + of a CDN. Defaults to `False`. +* `DEV_LISTEN_HOST`: For use with dev.py for development. Default is localhost +* `DEV_LISTEN_PORT`: For use with dev.py for development. Default is 5000 ## Getting Help diff --git a/docs/EL7.md b/docs/EL7.md index e09ece5a3..669116c04 100644 --- a/docs/EL7.md +++ b/docs/EL7.md @@ -150,7 +150,6 @@ class profile::puppetboard { puppetdb_key => "${ssl_dir}/private_keys/${puppetboard_certname}.pem", puppetdb_ssl_verify => "${ssl_dir}/certs/ca.pem", puppetdb_cert => "${ssl_dir}/certs/${puppetboard_certname}.pem", - reports_count => 40, } class { '::apache::mod::wsgi': diff --git a/puppetboard/docker_settings.py b/puppetboard/docker_settings.py index d1f6be127..5a210bc16 100644 --- a/puppetboard/docker_settings.py +++ b/puppetboard/docker_settings.py @@ -39,7 +39,7 @@ def coerce_bool(v, default): LOCALISE_TIMESTAMP = coerce_bool(os.getenv('LOCALISE_TIMESTAMP'), True) LOGLEVEL = os.getenv('LOGLEVEL', 'info') -NORMAL_TABLE_COUNT = int(os.getenv('REPORTS_COUNT', '100')) +NORMAL_TABLE_COUNT = int(os.getenv('NORMAL_TABLE_COUNT', '100')) LITTLE_TABLE_COUNT = int(os.getenv('LITTLE_TABLE_COUNT', '10')) TABLE_COUNT_DEF = "10,20,50,100,500" From e10d6c28cf5a075ef54d9c03bd6bc8440867145c Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 20 Mar 2021 20:37:03 +0100 Subject: [PATCH 04/10] This should fix all tests --- puppetboard/static/css/puppetboard.css | 4 +++ puppetboard/templates/index.html | 2 +- test/test_app.py | 40 +++++++++++++++++--------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index 711980583..46c2ff973 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -57,6 +57,10 @@ h1.ui.header.no-margin-bottom { background-color: #4572A7; } +.ui.header.population { + background-color: #2C3E50; +} + .ui.header.unreported { color: #3D96AE; } diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index b5ed6a262..5ff388106 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -51,7 +51,7 @@

unreported in the last {{ config.UNRESPONSIVE_HOURS }} hours

-

+

{{ metrics['num_nodes'] }} {% if metrics['num_nodes'] == 1 %} node {% else %} nodes {% endif %}

diff --git a/test/test_app.py b/test/test_app.py index f9669b9b2..b0082c7e6 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -169,10 +169,14 @@ def test_index_all(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '63' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '63' + assert vals[1].string == ' 6' + + vals2 = soup.find_all('h1', + {"class": "ui header population no-margin-bottom"}) + assert len(vals2) == 1 + assert '10' in vals2[0].string assert rv.status_code == 200 @@ -219,10 +223,14 @@ def test_index_all_puppetdb_v4(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '63' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '63' + assert vals[1].string == ' 6' + + vals2 = soup.find_all('h1', + {"class": "ui header population no-margin-bottom"}) + assert len(vals2) == 1 + assert '10' in vals2[0].string assert rv.status_code == 200 @@ -269,10 +277,14 @@ def test_index_all_puppetdb_v3(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '60' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '60' + assert vals[1].string == ' 6' + + vals2 = soup.find_all('h1', + {"class": "ui header population no-margin-bottom"}) + assert len(vals2) == 1 + assert '10' in vals2[0].string assert rv.status_code == 200 @@ -298,8 +310,8 @@ def test_index_division_by_zero(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[2].string == '0' + assert len(vals) == 2 + assert vals[1].string == '0' def test_offline_mode(client, mocker, From adddc985fc427baf86ce02d5bdbb68f20cf7e8ca Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 20 Mar 2021 20:46:59 +0100 Subject: [PATCH 05/10] Actually don't get the underlying data when UI elements are disabled --- puppetboard/app.py | 179 +++++++++++++++++++++------------------------ 1 file changed, 85 insertions(+), 94 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index f243d2aca..73a6423f2 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -136,11 +136,21 @@ def index(env): :type env: :obj:`string` """ envs = environments() + check_env(env, envs) + metrics = { 'num_nodes': 0, 'num_resources': 0, - 'avg_resources_node': 0} - check_env(env, envs) + 'avg_resources_node': 0, + } + nodes_overview = [] + stats = { + 'changed': 0, + 'unchanged': 0, + 'failed': 0, + 'unreported': 0, + 'noop': 0 + } if env == '*': query = app.config['OVERVIEW_FILTER'] @@ -153,21 +163,22 @@ def index(env): puppetdb.metric, "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), version=metric_version) - num_resources = get_or_abort( - puppetdb.metric, - "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), - version=metric_version) - metrics['num_nodes'] = num_nodes['Value'] - metrics['num_resources'] = num_resources['Value'] - try: - # Compute our own average because avg_resources_node['Value'] - # returns a string of the format "num_resources/num_nodes" - # example: "1234/9" instead of doing the division itself. - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources['Value'] / num_nodes['Value'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 + if app.config['RESOURCES_STATS_ENABLED']: + num_resources = get_or_abort( + puppetdb.metric, + "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), + version=metric_version) + + metrics['num_resources'] = num_resources['Value'] + try: + # Compute our own average because avg_resources_node['Value'] + # returns a string of the format "num_resources/num_nodes" + # example: "1234/9" instead of doing the division itself. + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources['Value'] / num_nodes['Value'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 else: query = AndOperator() query.add(EqualsOperator('catalog_environment', env)) @@ -179,55 +190,53 @@ def index(env): if app.config['OVERVIEW_FILTER'] is not None: query.add(app.config['OVERVIEW_FILTER']) - num_resources_query = ExtractOperator() - num_resources_query.add_field(FunctionOperator('count')) - num_resources_query.add_query(EqualsOperator("environment", env)) - num_nodes = get_or_abort( puppetdb._query, 'nodes', query=num_nodes_query) - num_resources = get_or_abort( - puppetdb._query, - 'resources', - query=num_resources_query) - metrics['num_nodes'] = num_nodes[0]['count'] - metrics['num_resources'] = num_resources[0]['count'] - try: - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources[0]['count'] / num_nodes[0]['count'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - - nodes = get_or_abort(puppetdb.nodes, - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - nodes_overview = [] - stats = { - 'changed': 0, - 'unchanged': 0, - 'failed': 0, - 'unreported': 0, - 'noop': 0 - } + metrics['num_nodes'] = num_nodes[0]['count'] - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - else: - stats['unchanged'] += 1 + if app.config['RESOURCES_STATS_ENABLED']: + + num_resources_query = ExtractOperator() + num_resources_query.add_field(FunctionOperator('count')) + num_resources_query.add_query(EqualsOperator("environment", env)) + + num_resources = get_or_abort( + puppetdb._query, + 'resources', + query=num_resources_query) + + metrics['num_resources'] = num_resources[0]['count'] + try: + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources[0]['count'] / num_nodes[0]['count'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if app.config['NODES_STATUS_DETAIL_ENABLED']: + + nodes = get_or_abort(puppetdb.nodes, + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) + + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + else: + stats['unchanged'] += 1 - if node.status != 'unchanged': - nodes_overview.append(node) + if node.status != 'unchanged': + nodes_overview.append(node) return render_template( 'index.html', @@ -626,50 +635,32 @@ def facts(env): """ envs = environments() check_env(env, envs) + facts = [] + order_by = '[{"field": "name", "order": "asc"}]' facts = get_or_abort(puppetdb.fact_names) - # we consider a column label to count for ~5 lines - column_label_height = 5 - - # 1 label per different letter and up to 3 more labels for letters spanning - # multiple columns. - column_label_count = 3 + len(set(map(lambda fact: fact[0].upper(), facts))) - - break_size = (len(facts) + column_label_count * column_label_height) / 4.0 - next_break = break_size - - facts_columns = [] - facts_current_column = [] - facts_current_letter = [] + facts_columns = [[]] letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size count = 0 - for fact in facts: count += 1 - if count > next_break: - next_break += break_size - if facts_current_letter: - facts_current_column.append(facts_current_letter) - if facts_current_column: - facts_columns.append(facts_current_column) - facts_current_column = [] - facts_current_letter = [] - letter = None - - if letter != fact[0].upper(): - if facts_current_letter: - facts_current_column.append(facts_current_letter) - facts_current_letter = [] + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset letter = fact[0].upper() - count += column_label_height - - facts_current_letter.append(fact) + letter_list = [] - if facts_current_letter: - facts_current_column.append(facts_current_letter) - if facts_current_column: - facts_columns.append(facts_current_column) + letter_list.append(fact) + facts_columns[-1].append(letter_list) return render_template('facts.html', facts_columns=facts_columns, @@ -921,7 +912,7 @@ def metrics(env): metrics.append(domain + ':' + prop) else: raise ValueError("Unknown metric version {} for database version {}" - .format(metric_version, db_version)) + .format(metric_version, database_version)) return render_template('metrics.html', metrics=sorted(metrics), From cdd00595277cd8b19b5ae9ba6eec3396a4724225 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sun, 21 Mar 2021 21:18:11 +0100 Subject: [PATCH 06/10] Fix style --- puppetboard/static/css/puppetboard.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index 46c2ff973..17012c9cf 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -58,7 +58,7 @@ h1.ui.header.no-margin-bottom { } .ui.header.population { - background-color: #2C3E50; + color: #2C3E50; } .ui.header.unreported { From c579f8684dc85dcc780cbae9c5a721d32d7e52d9 Mon Sep 17 00:00:00 2001 From: Sebastian Rakel Date: Wed, 6 Oct 2021 13:41:31 +0200 Subject: [PATCH 07/10] Use smaller query to get overall infos without pull the complete node infos --- puppetboard/app.py | 73 ++++++++++++++++++++++++-------- puppetboard/templates/index.html | 3 ++ test/test_app.py | 2 +- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 73a6423f2..1b9a6e8f5 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -19,6 +19,9 @@ LessEqualOperator, RegexOperator, GreaterEqualOperator) +from pypuppetdb.types import Node +from pypuppetdb.utils import json_to_datetime + from puppetboard.forms import ENABLED_QUERY_ENDPOINTS, QueryForm from puppetboard.utils import (get_or_abort, yield_or_stop, get_db_version, is_bool) @@ -152,6 +155,10 @@ def index(env): 'noop': 0 } + nodes = [] + + node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] + resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] if env == '*': query = app.config['OVERVIEW_FILTER'] @@ -164,7 +171,7 @@ def index(env): "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), version=metric_version) - if app.config['RESOURCES_STATS_ENABLED']: + if resource_stats_enabled: num_resources = get_or_abort( puppetdb.metric, "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), @@ -179,6 +186,9 @@ def index(env): (num_resources['Value'] / num_nodes['Value'])) except ZeroDivisionError: metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) else: query = AndOperator() query.add(EqualsOperator('catalog_environment', env)) @@ -197,8 +207,7 @@ def index(env): metrics['num_nodes'] = num_nodes[0]['count'] - if app.config['RESOURCES_STATS_ENABLED']: - + if resource_stats_enabled: num_resources_query = ExtractOperator() num_resources_query.add_field(FunctionOperator('count')) num_resources_query.add_query(EqualsOperator("environment", env)) @@ -215,26 +224,29 @@ def index(env): except ZeroDivisionError: metrics['avg_resources_node'] = 0 - if app.config['NODES_STATUS_DETAIL_ENABLED']: + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + if node_status_detail_enabled: nodes = get_or_abort(puppetdb.nodes, query=query, unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True, with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - else: - stats['unchanged'] += 1 + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + else: + stats['unchanged'] += 1 + if node_status_detail_enabled: if node.status != 'unchanged': nodes_overview.append(node) @@ -248,6 +260,33 @@ def index(env): ) +def get_node_status_summary(inner_query): + node_status_query = ExtractOperator() + node_status_query.add_field('certname') + node_status_query.add_field('report_timestamp') + node_status_query.add_field('latest_report_status') + node_status_query.add_query(inner_query) + + node_status = get_or_abort( + puppetdb._query, + 'nodes', + query=node_status_query + ) + + now = datetime.utcnow() + node_infos = [] + for node_state in node_status: + last_report = json_to_datetime(node_state['report_timestamp']) + last_report = last_report.replace(tzinfo=None) + unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) + certname = node_state['certname'] + + node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, + status_report=node_state['latest_report_status'])) + + return node_infos + + @app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//nodes') def nodes(env): @@ -635,8 +674,6 @@ def facts(env): """ envs = environments() check_env(env, envs) - facts = [] - order_by = '[{"field": "name", "order": "asc"}]' facts = get_or_abort(puppetdb.fact_names) facts_columns = [[]] @@ -912,7 +949,7 @@ def metrics(env): metrics.append(domain + ':' + prop) else: raise ValueError("Unknown metric version {} for database version {}" - .format(metric_version, database_version)) + .format(metric_version, db_version)) return render_template('metrics.html', metrics=sorted(metrics), diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index 5ff388106..586417359 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -61,10 +61,13 @@

{% if config.RESOURCES_STATS_ENABLED %}
+
+
+

{{metrics['num_resources']}}

diff --git a/test/test_app.py b/test/test_app.py index b0082c7e6..3c8fe3e11 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -781,7 +781,7 @@ def test_facts_view_empty_when_no_facts(client, searchable = soup.find('div', {'class': 'searchable'}) vals = searchable.find_all('div', {'class': 'column'}) - assert len(vals) == 0 + assert len(vals) == 1 def test_fact_view_with_graph(client, mocker, From 9377ed8b7d27cae50c2f4d0ce843675c742c961c Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Wed, 6 Oct 2021 18:10:17 +0200 Subject: [PATCH 08/10] Fix the tests --- test/test_app.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/test_app.py b/test/test_app.py index 3c8fe3e11..f21fc2fc3 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -173,11 +173,6 @@ def test_index_all(client, mocker, assert vals[0].string == '63' assert vals[1].string == ' 6' - vals2 = soup.find_all('h1', - {"class": "ui header population no-margin-bottom"}) - assert len(vals2) == 1 - assert '10' in vals2[0].string - assert rv.status_code == 200 @@ -227,11 +222,6 @@ def test_index_all_puppetdb_v4(client, mocker, assert vals[0].string == '63' assert vals[1].string == ' 6' - vals2 = soup.find_all('h1', - {"class": "ui header population no-margin-bottom"}) - assert len(vals2) == 1 - assert '10' in vals2[0].string - assert rv.status_code == 200 @@ -281,11 +271,6 @@ def test_index_all_puppetdb_v3(client, mocker, assert vals[0].string == '60' assert vals[1].string == ' 6' - vals2 = soup.find_all('h1', - {"class": "ui header population no-margin-bottom"}) - assert len(vals2) == 1 - assert '10' in vals2[0].string - assert rv.status_code == 200 From 5ff84347155b5d7781bed69a43527319f29bef55 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 8 Oct 2021 18:47:56 +0200 Subject: [PATCH 09/10] Refactor: split each tab into separate file --- puppetboard/app.py | 1130 +------------------------------------- puppetboard/catalogs.py | 165 ++++++ puppetboard/facts.py | 198 +++++++ puppetboard/forms.py | 36 -- puppetboard/index.py | 170 ++++++ puppetboard/inventory.py | 89 +++ puppetboard/metrics.py | 89 +++ puppetboard/nodes.py | 76 +++ puppetboard/query.py | 96 ++++ puppetboard/radiator.py | 106 ++++ puppetboard/reports.py | 223 ++++++++ puppetboard/utils.py | 4 + 12 files changed, 1225 insertions(+), 1157 deletions(-) create mode 100644 puppetboard/catalogs.py create mode 100644 puppetboard/facts.py delete mode 100644 puppetboard/forms.py create mode 100644 puppetboard/index.py create mode 100644 puppetboard/inventory.py create mode 100644 puppetboard/metrics.py create mode 100644 puppetboard/nodes.py create mode 100644 puppetboard/query.py create mode 100644 puppetboard/radiator.py create mode 100644 puppetboard/reports.py diff --git a/puppetboard/app.py b/puppetboard/app.py index 1b9a6e8f5..7054dc973 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -1,59 +1,21 @@ -from __future__ import absolute_import -from __future__ import unicode_literals +import logging +from datetime import datetime -from urllib.parse import unquote, unquote_plus, quote_plus -from datetime import datetime, timedelta -from itertools import tee -from distutils.util import strtobool from flask import ( - render_template, abort, url_for, - Response, stream_with_context, request, session, jsonify + render_template, abort, Response, request, jsonify ) - -import logging -import json - -from pypuppetdb.QueryBuilder import (ExtractOperator, AndOperator, - EqualsOperator, FunctionOperator, - NullOperator, OrOperator, - LessEqualOperator, RegexOperator, - GreaterEqualOperator) - -from pypuppetdb.types import Node -from pypuppetdb.utils import json_to_datetime - -from puppetboard.forms import ENABLED_QUERY_ENDPOINTS, QueryForm -from puppetboard.utils import (get_or_abort, yield_or_stop, - get_db_version, is_bool) -from puppetboard.dailychart import get_daily_reports_chart - -import commonmark +from pypuppetdb.QueryBuilder import (AndOperator, + EqualsOperator) from puppetboard.core import get_app, get_puppetdb, environments - +from puppetboard.dailychart import get_daily_reports_chart +from puppetboard.reports import REPORTS_COLUMNS +from puppetboard.utils import (get_or_abort) from puppetboard.version import __version__ -REPORTS_COLUMNS = [ - {'attr': 'end', 'filter': 'end_time', - 'name': 'End time', 'type': 'datetime'}, - {'attr': 'status', 'name': 'Status', 'type': 'status'}, - {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, - {'attr': 'version', 'filter': 'configuration_version', - 'name': 'Configuration version'}, - {'attr': 'agent_version', 'filter': 'puppet_version', - 'name': 'Agent version'}, -] - -CATALOGS_COLUMNS = [ - {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, - {'attr': 'catalog_timestamp', 'name': 'Compile Time'}, - {'attr': 'form', 'name': 'Compare'}, -] - app = get_app() -graph_facts = app.config['GRAPH_FACTS'] -numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) +numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) logging.basicConfig(level=numeric_level) log = logging.getLogger(__name__) @@ -87,14 +49,6 @@ def version(): return __version__ -def stream_template(template_name, **context): - app.update_template_context(context) - t = app.jinja_env.get_template(template_name) - rv = t.stream(context) - rv.enable_buffering(5) - return rv - - def check_env(env, envs): if env != '*' and env not in envs: abort(404) @@ -129,307 +83,6 @@ def now(format='%m/%d/%Y %H:%M:%S'): return dict(now=now) -@app.route('/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//') -def index(env): - """This view generates the index page and displays a set of metrics and - latest reports on nodes fetched from PuppetDB. - - :param env: Search for nodes in this (Catalog and Fact) environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - metrics = { - 'num_nodes': 0, - 'num_resources': 0, - 'avg_resources_node': 0, - } - nodes_overview = [] - stats = { - 'changed': 0, - 'unchanged': 0, - 'failed': 0, - 'unreported': 0, - 'noop': 0 - } - - nodes = [] - - node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] - resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] - if env == '*': - query = app.config['OVERVIEW_FILTER'] - - prefix = 'puppetlabs.puppetdb.population' - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - num_nodes = get_or_abort( - puppetdb.metric, - "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), - version=metric_version) - - if resource_stats_enabled: - num_resources = get_or_abort( - puppetdb.metric, - "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), - version=metric_version) - - metrics['num_resources'] = num_resources['Value'] - try: - # Compute our own average because avg_resources_node['Value'] - # returns a string of the format "num_resources/num_nodes" - # example: "1234/9" instead of doing the division itself. - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources['Value'] / num_nodes['Value'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - - if not node_status_detail_enabled: - nodes = get_node_status_summary(query) - else: - query = AndOperator() - query.add(EqualsOperator('catalog_environment', env)) - - num_nodes_query = ExtractOperator() - num_nodes_query.add_field(FunctionOperator('count')) - num_nodes_query.add_query(query) - - if app.config['OVERVIEW_FILTER'] is not None: - query.add(app.config['OVERVIEW_FILTER']) - - num_nodes = get_or_abort( - puppetdb._query, - 'nodes', - query=num_nodes_query) - - metrics['num_nodes'] = num_nodes[0]['count'] - - if resource_stats_enabled: - num_resources_query = ExtractOperator() - num_resources_query.add_field(FunctionOperator('count')) - num_resources_query.add_query(EqualsOperator("environment", env)) - - num_resources = get_or_abort( - puppetdb._query, - 'resources', - query=num_resources_query) - - metrics['num_resources'] = num_resources[0]['count'] - try: - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources[0]['count'] / num_nodes[0]['count'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - - if not node_status_detail_enabled: - nodes = get_node_status_summary(query) - - if node_status_detail_enabled: - nodes = get_or_abort(puppetdb.nodes, - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - else: - stats['unchanged'] += 1 - - if node_status_detail_enabled: - if node.status != 'unchanged': - nodes_overview.append(node) - - return render_template( - 'index.html', - metrics=metrics, - nodes=nodes_overview, - stats=stats, - envs=envs, - current_env=env - ) - - -def get_node_status_summary(inner_query): - node_status_query = ExtractOperator() - node_status_query.add_field('certname') - node_status_query.add_field('report_timestamp') - node_status_query.add_field('latest_report_status') - node_status_query.add_query(inner_query) - - node_status = get_or_abort( - puppetdb._query, - 'nodes', - query=node_status_query - ) - - now = datetime.utcnow() - node_infos = [] - for node_state in node_status: - last_report = json_to_datetime(node_state['report_timestamp']) - last_report = last_report.replace(tzinfo=None) - unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) - certname = node_state['certname'] - - node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, - status_report=node_state['latest_report_status'])) - - return node_infos - - -@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//nodes') -def nodes(env): - """Fetch all (active) nodes from PuppetDB and stream a table displaying - those nodes. - - Downside of the streaming aproach is that since we've already sent our - headers we can't abort the request if we detect an error. Because of this - we'll end up with an empty table instead because of how yield_or_stop - works. Once pagination is in place we can change this but we'll need to - provide a search feature instead. - - :param env: Search for nodes in this (Catalog and Fact) environment - :type env: :obj:`string` - """ - envs = environments() - status_arg = request.args.get('status', '') - check_env(env, envs) - - query = AndOperator() - - if env != '*': - query.add(EqualsOperator("catalog_environment", env)) - - if status_arg in ['failed', 'changed', 'unchanged']: - query.add(EqualsOperator('latest_report_status', status_arg)) - elif status_arg == 'unreported': - unreported = datetime.utcnow() - unreported = (unreported - - timedelta(hours=app.config['UNRESPONSIVE_HOURS'])) - unreported = unreported.replace(microsecond=0).isoformat() - - unrep_query = OrOperator() - unrep_query.add(NullOperator('report_timestamp', True)) - unrep_query.add(LessEqualOperator('report_timestamp', unreported)) - - query.add(unrep_query) - - if len(query.operations) == 0: - query = None - - nodelist = puppetdb.nodes( - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - nodes = [] - for node in yield_or_stop(nodelist): - if status_arg: - if node.status == status_arg: - nodes.append(node) - else: - nodes.append(node) - return Response(stream_with_context( - stream_template('nodes.html', - nodes=nodes, - envs=envs, - current_env=env))) - - -def inventory_facts(): - # a list of facts descriptions to go in table header - headers = [] - # a list of inventory fact names - fact_names = [] - - # load the list of items/facts we want in our inventory - try: - inv_facts = app.config['INVENTORY_FACTS'] - except KeyError: - inv_facts = [('Hostname', 'fqdn'), - ('IP Address', 'ipaddress'), - ('OS', 'lsbdistdescription'), - ('Architecture', 'hardwaremodel'), - ('Kernel Version', 'kernelrelease')] - - # generate a list of descriptions and a list of fact names - # from the list of tuples inv_facts. - for desc, name in inv_facts: - headers.append(desc) - fact_names.append(name) - - return headers, fact_names - - -@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//inventory') -def inventory(env): - """Fetch all (active) nodes from PuppetDB and stream a table displaying - those nodes along with a set of facts about them. - - :param env: Search for facts in this environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - headers, fact_names = inventory_facts() - - return render_template( - 'inventory.html', - envs=envs, - current_env=env, - fact_headers=headers) - - -@app.route('/inventory/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//inventory/json') -def inventory_ajax(env): - """Backend endpoint for inventory table""" - draw = int(request.args.get('draw', 0)) - - envs = environments() - check_env(env, envs) - headers, fact_names = inventory_facts() - - query = AndOperator() - fact_query = OrOperator() - fact_query.add([EqualsOperator("name", name) for name in fact_names]) - query.add(fact_query) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - facts = puppetdb.facts(query=query) - - fact_data = {} - for fact in facts: - if fact.node not in fact_data: - fact_data[fact.node] = {} - fact_data[fact.node][fact.name] = fact.value - - total = len(fact_data) - - return render_template( - 'inventory.json.tpl', - draw=draw, - total=total, - total_filtered=total, - fact_data=fact_data, - columns=fact_names) - - @app.route('/node/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//node/') @@ -460,771 +113,6 @@ def node(env, node_name): columns=REPORTS_COLUMNS[:2]) -@app.route('/reports', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node_name': None}) -@app.route('//reports', defaults={'node_name': None}) -@app.route('/reports/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//reports/') -def reports(env, node_name): - """Query and Return JSON data to reports Jquery datatable - - :param env: Search for all reports in this environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - return render_template( - 'reports.html', - envs=envs, - current_env=env, - node_name=node_name, - columns=REPORTS_COLUMNS) - - -@app.route('/reports/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node_name': None}) -@app.route('//reports/json', defaults={'node_name': None}) -@app.route('/reports//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//reports//json') -def reports_ajax(env, node_name): - """Query and Return JSON data to reports Jquery datatable - - :param env: Search for all reports in this environment - :type env: :obj:`string` - """ - draw = int(request.args.get('draw', 0)) - start = int(request.args.get('start', 0)) - length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) - paging_args = {'limit': length, 'offset': start} - search_arg = request.args.get('search[value]') - order_column = int(request.args.get('order[0][column]', 0)) - order_filter = REPORTS_COLUMNS[order_column].get( - 'filter', REPORTS_COLUMNS[order_column]['attr']) - order_dir = request.args.get('order[0][dir]', 'desc') - order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) - status_args = request.args.get('columns[1][search][value]', '').split('|') - date_args = request.args.get('columns[0][search][value]', '') - max_col = len(REPORTS_COLUMNS) - for i in range(len(REPORTS_COLUMNS)): - if request.args.get("columns[%s][data]" % i, None): - max_col = i + 1 - - envs = environments() - check_env(env, envs) - reports_query = AndOperator() - - if env != '*': - reports_query.add(EqualsOperator("environment", env)) - - if node_name: - reports_query.add(EqualsOperator("certname", node_name)) - - if search_arg: - search_query = OrOperator() - search_query.add(RegexOperator("certname", r"%s" % search_arg)) - search_query.add(RegexOperator("puppet_version", r"%s" % search_arg)) - search_query.add(RegexOperator( - "configuration_version", r"%s" % search_arg)) - reports_query.add(search_query) - - if date_args: - dates = json.loads(date_args) - - if len(dates) > 0: - date_query = AndOperator() - - if 'min' in dates: - date_query.add(GreaterEqualOperator('end_time', dates['min'])) - - if 'max' in dates: - date_query.add(LessEqualOperator('end_time', dates['max'])) - - reports_query.add(date_query) - - status_query = OrOperator() - for status_arg in status_args: - if status_arg in ['failed', 'changed', 'unchanged']: - arg_query = AndOperator() - arg_query.add(EqualsOperator('status', status_arg)) - arg_query.add(EqualsOperator('noop', False)) - status_query.add(arg_query) - if status_arg == 'unchanged': - arg_query = AndOperator() - arg_query.add(EqualsOperator('noop', True)) - arg_query.add(EqualsOperator('noop_pending', False)) - status_query.add(arg_query) - elif status_arg == 'noop': - arg_query = AndOperator() - arg_query.add(EqualsOperator('noop', True)) - arg_query.add(EqualsOperator('noop_pending', True)) - status_query.add(arg_query) - - if len(status_query.operations) == 0: - if len(reports_query.operations) == 0: - reports_query = None - else: - reports_query.add(status_query) - - if status_args[0] != 'none': - reports = get_or_abort( - puppetdb.reports, - query=reports_query, - order_by=order_args, - include_total=True, - **paging_args) - reports, reports_events = tee(reports) - total = None - else: - reports = [] - reports_events = [] - total = 0 - - # Convert metrics to relational dict - metrics = {} - for report in reports_events: - if total is None: - total = puppetdb.total - - metrics[report.hash_] = {} - for m in report.metrics: - if m['category'] not in metrics[report.hash_]: - metrics[report.hash_][m['category']] = {} - metrics[report.hash_][m['category']][m['name']] = m['value'] - - if total is None: - total = 0 - - return render_template( - 'reports.json.tpl', - draw=draw, - total=total, - total_filtered=total, - reports=reports, - metrics=metrics, - envs=envs, - current_env=env, - columns=REPORTS_COLUMNS[:max_col]) - - -@app.route('/report//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//report//') -def report(env, node_name, report_id): - """Displays a single report including all the events associated with that - report and their status. - - The report_id may be the puppetdb's report hash or the - configuration_version. This allows for better integration - into puppet-hipchat. - - :param env: Search for reports in this environment - :type env: :obj:`string` - :param node_name: Find the reports whose certname match this value - :type node_name: :obj:`string` - :param report_id: The hash or the configuration_version of the desired - report - :type report_id: :obj:`string` - """ - envs = environments() - check_env(env, envs) - query = AndOperator() - report_id_query = OrOperator() - - report_id_query.add(EqualsOperator("hash", report_id)) - report_id_query.add(EqualsOperator("configuration_version", report_id)) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - query.add(EqualsOperator("certname", node_name)) - query.add(report_id_query) - - reports = puppetdb.reports(query=query) - - try: - report = next(reports) - except StopIteration: - abort(404) - - report.version = commonmark.commonmark(report.version) - - return render_template( - 'report.html', - report=report, - events=yield_or_stop(report.events()), - logs=report.logs, - metrics=report.metrics, - envs=envs, - current_env=env) - - -@app.route('/facts', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//facts') -def facts(env): - """Displays an alphabetical list of all facts currently known to - PuppetDB. - - :param env: Serves no purpose for this function, only for consistency's - sake - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - facts = get_or_abort(puppetdb.fact_names) - - facts_columns = [[]] - letter = None - letter_list = None - break_size = (len(facts) / 4) + 1 - next_break = break_size - count = 0 - for fact in facts: - count += 1 - - if letter != fact[0].upper() or not letter: - if count > next_break: - # Create a new column - facts_columns.append([]) - next_break += break_size - if letter_list: - facts_columns[-1].append(letter_list) - # Reset - letter = fact[0].upper() - letter_list = [] - - letter_list.append(fact) - facts_columns[-1].append(letter_list) - - return render_template('facts.html', - facts_columns=facts_columns, - envs=envs, - current_env=env) - - -@app.route('/fact/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) -@app.route('//fact/', defaults={'value': None}) -@app.route('/fact//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact//') -def fact(env, fact, value): - """Fetches the specific fact(/value) from PuppetDB and displays per - node for which this fact is known. - - :param env: Searches for facts in this environment - :type env: :obj:`string` - :param fact: Find all facts with this name - :type fact: :obj:`string` - :param value: Find all facts with this value - :type value: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - render_graph = False - if fact in graph_facts and not value: - render_graph = True - - value_safe = value - if value is not None: - value_safe = unquote_plus(value) - - return render_template( - 'fact.html', - fact=fact, - value=value, - value_safe=value_safe, - render_graph=render_graph, - envs=envs, - current_env=env) - - -@app.route('/fact//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node': None, 'value': None}) -@app.route('//fact//json', defaults={'node': None, 'value': None}) -@app.route('/fact///json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) -@app.route('/fact///json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) -@app.route('//fact///json', defaults={'node': None}) -@app.route('/node//facts/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'fact': None, 'value': None}) -@app.route('//node//facts/json', - defaults={'fact': None, 'value': None}) -def fact_ajax(env, node, fact, value): - """Fetches the specific facts matching (node/fact/value) from PuppetDB and - return a JSON table - - :param env: Searches for facts in this environment - :type env: :obj:`string` - :param node: Find all facts for this node - :type node: :obj:`string` - :param fact: Find all facts with this name - :type fact: :obj:`string` - :param value: Filter facts whose value is equal to this - :type value: :obj:`string` - """ - draw = int(request.args.get('draw', 0)) - - envs = environments() - check_env(env, envs) - - render_graph = False - if fact in graph_facts and not value and not node: - render_graph = True - - query = AndOperator() - if node: - query.add(EqualsOperator("certname", node)) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - if len(query.operations) == 0: - query = None - - # Generator needs to be converted (graph / total) - try: - value = int(value) - except ValueError: - if value is not None and query is not None: - if is_bool(value): - query.add(EqualsOperator('value', bool(strtobool(value)))) - else: - query.add(EqualsOperator('value', unquote_plus(value))) - except TypeError: - pass - - facts = [f for f in get_or_abort( - puppetdb.facts, - name=fact, - query=query)] - - total = len(facts) - - counts = {} - json = { - 'draw': draw, - 'recordsTotal': total, - 'recordsFiltered': total, - 'data': []} - - for fact_h in facts: - line = [] - if not fact: - line.append(fact_h.name) - if not node: - line.append('{1}'.format( - url_for('node', env=env, node_name=fact_h.node), - fact_h.node)) - if not value: - fact_value = fact_h.value - if isinstance(fact_value, str): - fact_value = quote_plus(fact_h.value) - - line.append('{1}'.format( - url_for( - 'fact', env=env, fact=fact_h.name, value=fact_value), - fact_h.value)) - - json['data'].append(line) - - if render_graph: - if fact_h.value not in counts: - counts[fact_h.value] = 0 - counts[fact_h.value] += 1 - - if render_graph: - json['chart'] = [ - {"label": "{0}".format(k).replace('\n', ' '), - "value": counts[k]} - for k in sorted(counts, key=lambda k: counts[k], reverse=True)] - - return jsonify(json) - - -@app.route('/query', methods=('GET', 'POST'), - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//query', methods=('GET', 'POST')) -def query(env): - """Allows to execute raw, user created querries against PuppetDB. This is - currently highly experimental and explodes in interesting ways since none - of the possible exceptions are being handled just yet. This will return - the JSON of the response or a message telling you what whent wrong / - why nothing was returned. - - :param env: Serves no purpose for the query data but is required for the - select field in the environment block - :type env: :obj:`string` - """ - if not app.config['ENABLE_QUERY']: - log.warn('Access to query interface disabled by administrator.') - abort(403) - - envs = environments() - check_env(env, envs) - - form = QueryForm(meta={ - 'csrf_secret': app.config['SECRET_KEY'], - 'csrf_context': session}) - if form.validate_on_submit(): - if form.endpoints.data not in ENABLED_QUERY_ENDPOINTS: - log.warn('Access to query endpoint %s disabled by administrator.', - form.endpoints.data) - abort(403) - - if form.endpoints.data == 'pql': - query = form.query.data - elif form.query.data[0] == '[': - query = form.query.data - else: - query = '[{0}]'.format(form.query.data) - - result = get_or_abort( - puppetdb._query, - form.endpoints.data, - query=query) - return render_template('query.html', - form=form, - result=result, - envs=envs, - current_env=env) - return render_template('query.html', - form=form, - envs=envs, - current_env=env) - - -@app.route('/metrics', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//metrics') -def metrics(env): - """Lists all available metrics that PuppetDB is aware of. - - :param env: While this parameter serves no function purpose it is required - for the environments template block - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - if metric_version == 'v1': - mbeans = get_or_abort(puppetdb._query, 'mbean') - metrics = list(mbeans.keys()) - elif metric_version == 'v2': - # the list response is a dict in the format: - # { - # "domain1": { - # "property1": { - # ... - # } - # }, - # "domain2": { - # "property2": { - # ... - # } - # } - # } - # The MBean names are the combination of the domain and the properties - # with a ":" in between, example: - # domain1:property1 - # domain2:property2 - # reference: https://jolokia.org/reference/html/protocol.html#list - metrics_domains = get_or_abort(puppetdb.metric) - metrics = [] - # get all of the domains - for domain in list(metrics_domains.keys()): - # iterate over all of the properties in this domain - properties = list(metrics_domains[domain].keys()) - for prop in properties: - # combine the current domain and each property with - # a ":" in between - metrics.append(domain + ':' + prop) - else: - raise ValueError("Unknown metric version {} for database version {}" - .format(metric_version, db_version)) - - return render_template('metrics.html', - metrics=sorted(metrics), - envs=envs, - current_env=env) - - -@app.route('/metric/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//metric/') -def metric(env, metric): - """Lists all information about the metric of the given name. - - :param env: While this parameter serves no function purpose it is required - for the environments template block - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - name = unquote(metric) - metric = get_or_abort(puppetdb.metric, metric, version=metric_version) - return render_template( - 'metric.html', - name=name, - metric=sorted(metric.items()), - envs=envs, - current_env=env) - - -@app.route('/catalogs', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'compare': None}) -@app.route('//catalogs', defaults={'compare': None}) -@app.route('/catalogs/compare/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare/') -def catalogs(env, compare): - """Lists all nodes with a compiled catalog. - - :param env: Find the nodes with this catalog_environment value - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if not app.config['ENABLE_CATALOG']: - log.warning('Access to catalog interface disabled by administrator') - abort(403) - - return render_template( - 'catalogs.html', - compare=compare, - columns=CATALOGS_COLUMNS, - envs=envs, - current_env=env) - - -@app.route('/catalogs/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'compare': None}) -@app.route('//catalogs/json', defaults={'compare': None}) -@app.route('/catalogs/compare//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare//json') -def catalogs_ajax(env, compare): - """Server data to catalogs as JSON to Jquery datatables - """ - draw = int(request.args.get('draw', 0)) - start = int(request.args.get('start', 0)) - length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) - paging_args = {'limit': length, 'offset': start} - search_arg = request.args.get('search[value]') - order_column = int(request.args.get('order[0][column]', 0)) - order_filter = CATALOGS_COLUMNS[order_column].get( - 'filter', CATALOGS_COLUMNS[order_column]['attr']) - order_dir = request.args.get('order[0][dir]', 'asc') - order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) - - envs = environments() - check_env(env, envs) - - query = AndOperator() - if env != '*': - query.add(EqualsOperator("catalog_environment", env)) - if search_arg: - query.add(RegexOperator("certname", r"%s" % search_arg)) - query.add(NullOperator("catalog_timestamp", False)) - - nodes = get_or_abort(puppetdb.nodes, - query=query, - include_total=True, - order_by=order_args, - **paging_args) - - catalog_list = [] - total = None - for node in nodes: - if total is None: - total = puppetdb.total - - catalog_list.append({ - 'certname': node.name, - 'catalog_timestamp': node.catalog_timestamp, - 'form': compare, - }) - - if total is None: - total = 0 - - return render_template( - 'catalogs.json.tpl', - total=total, - total_filtered=total, - draw=draw, - columns=CATALOGS_COLUMNS, - catalogs=catalog_list, - envs=envs, - current_env=env) - - -@app.route('/catalog/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalog/') -def catalog_node(env, node_name): - """Fetches from PuppetDB the compiled catalog of a given node. - - :param env: Find the catalog with this environment value - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if app.config['ENABLE_CATALOG']: - catalog = get_or_abort(puppetdb.catalog, - node=node_name) - return render_template('catalog.html', - catalog=catalog, - envs=envs, - current_env=env) - else: - log.warn('Access to catalog interface disabled by administrator') - abort(403) - - -@app.route('/catalogs/compare/...', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare/...') -def catalog_compare(env, compare, against): - """Compares the catalog of one node, parameter compare, with that of - with that of another node, parameter against. - - :param env: Ensure that the 2 catalogs are in the same environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if app.config['ENABLE_CATALOG']: - compare_cat = get_or_abort(puppetdb.catalog, - node=compare) - against_cat = get_or_abort(puppetdb.catalog, - node=against) - - return render_template('catalog_compare.html', - compare=compare_cat, - against=against_cat, - envs=envs, - current_env=env) - else: - log.warn('Access to catalog interface disabled by administrator') - abort(403) - - -@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//radiator') -def radiator(env): - """This view generates a simplified monitoring page - akin to the radiator view in puppet dashboard - """ - envs = environments() - check_env(env, envs) - - if env == '*': - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - query = None - metrics = get_or_abort( - puppetdb.metric, - 'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type, - version=metric_version) - num_nodes = metrics['Value'] - else: - query = AndOperator() - metric_query = ExtractOperator() - - query.add(EqualsOperator("catalog_environment", env)) - metric_query.add_field(FunctionOperator('count')) - metric_query.add_query(query) - - metrics = get_or_abort( - puppetdb._query, - 'nodes', - query=metric_query) - num_nodes = metrics[0]['count'] - - nodes = puppetdb.nodes( - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True - ) - - stats = { - 'changed_percent': 0, - 'changed': 0, - 'failed_percent': 0, - 'failed': 0, - 'noop_percent': 0, - 'noop': 0, - 'skipped_percent': 0, - 'skipped': 0, - 'unchanged_percent': 0, - 'unchanged': 0, - 'unreported_percent': 0, - 'unreported': 0, - } - - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - elif node.status == 'skipped': - stats['skipped'] += 1 - else: - stats['unchanged'] += 1 - - try: - stats['changed_percent'] = int(100 * (stats['changed'] / - float(num_nodes))) - stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) - stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) - stats['skipped_percent'] = int(100 * (stats['skipped'] / - float(num_nodes))) - stats['unchanged_percent'] = int(100 * (stats['unchanged'] / - float(num_nodes))) - stats['unreported_percent'] = int(100 * (stats['unreported'] / - float(num_nodes))) - except ZeroDivisionError: - stats['changed_percent'] = 0 - stats['failed_percent'] = 0 - stats['noop_percent'] = 0 - stats['skipped_percent'] = 0 - stats['unchanged_percent'] = 0 - stats['unreported_percent'] = 0 - - if ('Accept' in request.headers and - request.headers["Accept"] == 'application/json'): - return jsonify(**stats) - - return render_template( - 'radiator.html', - stats=stats, - total=num_nodes - ) - - @app.route('/daily_reports_chart.json', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//daily_reports_chart.json') diff --git a/puppetboard/catalogs.py b/puppetboard/catalogs.py new file mode 100644 index 000000000..3ebdb90ba --- /dev/null +++ b/puppetboard/catalogs.py @@ -0,0 +1,165 @@ +import logging + +from flask import render_template, request +from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, RegexOperator, NullOperator +from werkzeug.exceptions import abort + +import puppetboard.nodes +from puppetboard.app import app, check_env, puppetdb +from puppetboard.core import environments +from puppetboard.utils import get_or_abort + +numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) +logging.basicConfig(level=numeric_level) +log = logging.getLogger(__name__) + + +@app.route('/catalogs', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'compare': None}) +@app.route('//catalogs', defaults={'compare': None}) +@app.route('/catalogs/compare/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare/') +def catalogs(env, compare): + """Lists all nodes with a compiled catalog. + + :param env: Find the nodes with this catalog_environment value + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if not app.config['ENABLE_CATALOG']: + log.warning('Access to catalog interface disabled by administrator') + abort(403) + + return render_template( + 'catalogs.html', + compare=compare, + columns=CATALOGS_COLUMNS, + envs=envs, + current_env=env) + + +@app.route('/catalogs/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'compare': None}) +@app.route('//catalogs/json', defaults={'compare': None}) +@app.route('/catalogs/compare//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare//json') +def catalogs_ajax(env, compare): + """Server data to catalogs as JSON to Jquery datatables + """ + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) + paging_args = {'limit': length, 'offset': start} + search_arg = request.args.get('search[value]') + order_column = int(request.args.get('order[0][column]', 0)) + order_filter = CATALOGS_COLUMNS[order_column].get( + 'filter', CATALOGS_COLUMNS[order_column]['attr']) + order_dir = request.args.get('order[0][dir]', 'asc') + order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) + + envs = environments() + check_env(env, envs) + + query = AndOperator() + if env != '*': + query.add(EqualsOperator("catalog_environment", env)) + if search_arg: + query.add(RegexOperator("certname", r"%s" % search_arg)) + query.add(NullOperator("catalog_timestamp", False)) + + nodes = get_or_abort(puppetboard.nodes.nodes, + query=query, + include_total=True, + order_by=order_args, + **paging_args) + + catalog_list = [] + total = None + for node in nodes: + if total is None: + total = puppetdb.total + + catalog_list.append({ + 'certname': node.name, + 'catalog_timestamp': node.catalog_timestamp, + 'form': compare, + }) + + if total is None: + total = 0 + + return render_template( + 'catalogs.json.tpl', + total=total, + total_filtered=total, + draw=draw, + columns=CATALOGS_COLUMNS, + catalogs=catalog_list, + envs=envs, + current_env=env) + + +@app.route('/catalog/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalog/') +def catalog_node(env, node_name): + """Fetches from PuppetDB the compiled catalog of a given node. + + :param env: Find the catalog with this environment value + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if app.config['ENABLE_CATALOG']: + catalog = get_or_abort(puppetdb.catalog, + node=node_name) + return render_template('catalog.html', + catalog=catalog, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + + +@app.route('/catalogs/compare/...', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare/...') +def catalog_compare(env, compare, against): + """Compares the catalog of one node, parameter compare, with that of + with that of another node, parameter against. + + :param env: Ensure that the 2 catalogs are in the same environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if app.config['ENABLE_CATALOG']: + compare_cat = get_or_abort(puppetdb.catalog, + node=compare) + against_cat = get_or_abort(puppetdb.catalog, + node=against) + + return render_template('catalog_compare.html', + compare=compare_cat, + against=against_cat, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + + +CATALOGS_COLUMNS = [ + {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, + {'attr': 'catalog_timestamp', 'name': 'Compile Time'}, + {'attr': 'form', 'name': 'Compare'}, +] \ No newline at end of file diff --git a/puppetboard/facts.py b/puppetboard/facts.py new file mode 100644 index 000000000..0728dd9c2 --- /dev/null +++ b/puppetboard/facts.py @@ -0,0 +1,198 @@ +from distutils.util import strtobool +from urllib.parse import unquote_plus, quote_plus + +from flask import render_template, request, url_for, jsonify +from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator + +from puppetboard.app import app, check_env, puppetdb +from puppetboard.core import environments +from puppetboard.utils import get_or_abort, is_bool + +graph_facts = app.config['GRAPH_FACTS'] + + +@app.route('/facts', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//facts') +def facts(env): + """Displays an alphabetical list of all facts currently known to + PuppetDB. + + :param env: Serves no purpose for this function, only for consistency's + sake + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + facts = get_or_abort(puppetdb.fact_names) + + facts_columns = [[]] + letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size + count = 0 + for fact in facts: + count += 1 + + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset + letter = fact[0].upper() + letter_list = [] + + letter_list.append(fact) + facts_columns[-1].append(letter_list) + + return render_template('facts.html', + facts_columns=facts_columns, + envs=envs, + current_env=env) + + +@app.route('/fact/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) +@app.route('//fact/', defaults={'value': None}) +@app.route('/fact//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//fact//') +def fact(env, fact, value): + """Fetches the specific fact(/value) from PuppetDB and displays per + node for which this fact is known. + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Find all facts with this value + :type value: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + render_graph = False + if fact in graph_facts and not value: + render_graph = True + + value_safe = value + if value is not None: + value_safe = unquote_plus(value) + + return render_template( + 'fact.html', + fact=fact, + value=value, + value_safe=value_safe, + render_graph=render_graph, + envs=envs, + current_env=env) + + +@app.route('/fact//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node': None, 'value': None}) +@app.route('//fact//json', defaults={'node': None, 'value': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('//fact///json', defaults={'node': None}) +@app.route('/node//facts/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'fact': None, 'value': None}) +@app.route('//node//facts/json', + defaults={'fact': None, 'value': None}) +def fact_ajax(env, node, fact, value): + """Fetches the specific facts matching (node/fact/value) from PuppetDB and + return a JSON table + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param node: Find all facts for this node + :type node: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Filter facts whose value is equal to this + :type value: :obj:`string` + """ + draw = int(request.args.get('draw', 0)) + + envs = environments() + check_env(env, envs) + + render_graph = False + if fact in graph_facts and not value and not node: + render_graph = True + + query = AndOperator() + if node: + query.add(EqualsOperator("certname", node)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + if len(query.operations) == 0: + query = None + + # Generator needs to be converted (graph / total) + try: + value = int(value) + except ValueError: + if value is not None and query is not None: + if is_bool(value): + query.add(EqualsOperator('value', bool(strtobool(value)))) + else: + query.add(EqualsOperator('value', unquote_plus(value))) + except TypeError: + pass + + facts = [f for f in get_or_abort( + puppetdb.facts, + name=fact, + query=query)] + + total = len(facts) + + counts = {} + json = { + 'draw': draw, + 'recordsTotal': total, + 'recordsFiltered': total, + 'data': []} + + for fact_h in facts: + line = [] + if not fact: + line.append(fact_h.name) + if not node: + line.append('{1}'.format( + url_for('node', env=env, node_name=fact_h.node), + fact_h.node)) + if not value: + fact_value = fact_h.value + if isinstance(fact_value, str): + fact_value = quote_plus(fact_h.value) + + line.append('{1}'.format( + url_for( + 'fact', env=env, fact=fact_h.name, value=fact_value), + fact_h.value)) + + json['data'].append(line) + + if render_graph: + if fact_h.value not in counts: + counts[fact_h.value] = 0 + counts[fact_h.value] += 1 + + if render_graph: + json['chart'] = [ + {"label": "{0}".format(k).replace('\n', ' '), + "value": counts[k]} + for k in sorted(counts, key=lambda k: counts[k], reverse=True)] + + return jsonify(json) \ No newline at end of file diff --git a/puppetboard/forms.py b/puppetboard/forms.py deleted file mode 100644 index e7f969171..000000000 --- a/puppetboard/forms.py +++ /dev/null @@ -1,36 +0,0 @@ -from collections import OrderedDict - -from flask_wtf import FlaskForm -from wtforms import (BooleanField, SelectField, TextAreaField, validators) - -from puppetboard.core import get_app - -app = get_app() -QUERY_ENDPOINTS = OrderedDict([ - # PuppetDB API endpoint, Form name - ('pql', 'PQL'), - ('nodes', 'Nodes'), - ('resources', 'Resources'), - ('facts', 'Facts'), - ('factsets', 'Fact Sets'), - ('fact-paths', 'Fact Paths'), - ('fact-contents', 'Fact Contents'), - ('reports', 'Reports'), - ('events', 'Events'), - ('catalogs', 'Catalogs'), - ('edges', 'Edges'), - ('environments', 'Environments'), -]) -ENABLED_QUERY_ENDPOINTS = app.config.get( - 'ENABLED_QUERY_ENDPOINTS', list(QUERY_ENDPOINTS.keys())) - - -class QueryForm(FlaskForm): - """The form used to allow freeform queries to be executed against - PuppetDB.""" - query = TextAreaField('Query', [validators.DataRequired( - message='A query is required.')]) - endpoints = SelectField('API endpoint', choices=[ - (key, value) for key, value in QUERY_ENDPOINTS.items() - if key in ENABLED_QUERY_ENDPOINTS], default='pql') - rawjson = BooleanField('Raw JSON') diff --git a/puppetboard/index.py b/puppetboard/index.py new file mode 100644 index 000000000..6d02b2911 --- /dev/null +++ b/puppetboard/index.py @@ -0,0 +1,170 @@ +from datetime import datetime, timedelta + +from flask import render_template +from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, ExtractOperator, FunctionOperator +from pypuppetdb.types import Node +from pypuppetdb.utils import json_to_datetime + +import puppetboard.metrics +import puppetboard.nodes +from puppetboard.app import app, check_env, puppetdb, metric_params, node +from puppetboard.core import environments +from puppetboard.utils import get_db_version, get_or_abort + + +@app.route('/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//') +def index(env): + """This view generates the index page and displays a set of metrics and + latest reports on nodes fetched from PuppetDB. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + metrics = { + 'num_nodes': 0, + 'num_resources': 0, + 'avg_resources_node': 0, + } + nodes_overview = [] + stats = { + 'changed': 0, + 'unchanged': 0, + 'failed': 0, + 'unreported': 0, + 'noop': 0 + } + + nodes = [] + + node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] + resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] + if env == '*': + query = app.config['OVERVIEW_FILTER'] + + prefix = 'puppetlabs.puppetdb.population' + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + num_nodes = get_or_abort( + puppetboard.metrics.metric, + "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), + version=metric_version) + + if resource_stats_enabled: + num_resources = get_or_abort( + puppetboard.metrics.metric, + "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), + version=metric_version) + + metrics['num_resources'] = num_resources['Value'] + try: + # Compute our own average because avg_resources_node['Value'] + # returns a string of the format "num_resources/num_nodes" + # example: "1234/9" instead of doing the division itself. + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources['Value'] / num_nodes['Value'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + else: + query = AndOperator() + query.add(EqualsOperator('catalog_environment', env)) + + num_nodes_query = ExtractOperator() + num_nodes_query.add_field(FunctionOperator('count')) + num_nodes_query.add_query(query) + + if app.config['OVERVIEW_FILTER'] is not None: + query.add(app.config['OVERVIEW_FILTER']) + + num_nodes = get_or_abort( + puppetdb._query, + 'nodes', + query=num_nodes_query) + + metrics['num_nodes'] = num_nodes[0]['count'] + + if resource_stats_enabled: + num_resources_query = ExtractOperator() + num_resources_query.add_field(FunctionOperator('count')) + num_resources_query.add_query(EqualsOperator("environment", env)) + + num_resources = get_or_abort( + puppetdb._query, + 'resources', + query=num_resources_query) + + metrics['num_resources'] = num_resources[0]['count'] + try: + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources[0]['count'] / num_nodes[0]['count'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + + if node_status_detail_enabled: + nodes = get_or_abort(puppetboard.nodes.nodes, + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) + + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + else: + stats['unchanged'] += 1 + + if node_status_detail_enabled: + if node.status != 'unchanged': + nodes_overview.append(node) + + return render_template( + 'index.html', + metrics=metrics, + nodes=nodes_overview, + stats=stats, + envs=envs, + current_env=env + ) + + +def get_node_status_summary(inner_query): + node_status_query = ExtractOperator() + node_status_query.add_field('certname') + node_status_query.add_field('report_timestamp') + node_status_query.add_field('latest_report_status') + node_status_query.add_query(inner_query) + + node_status = get_or_abort( + puppetdb._query, + 'nodes', + query=node_status_query + ) + + now = datetime.utcnow() + node_infos = [] + for node_state in node_status: + last_report = json_to_datetime(node_state['report_timestamp']) + last_report = last_report.replace(tzinfo=None) + unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) + certname = node_state['certname'] + + node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, + status_report=node_state['latest_report_status'])) + + return node_infos \ No newline at end of file diff --git a/puppetboard/inventory.py b/puppetboard/inventory.py new file mode 100644 index 000000000..61f9a7c13 --- /dev/null +++ b/puppetboard/inventory.py @@ -0,0 +1,89 @@ +from flask import render_template, request +from pypuppetdb.QueryBuilder import AndOperator, OrOperator, EqualsOperator + +import puppetboard.facts +from puppetboard.app import app, check_env +from puppetboard.core import environments + + +def inventory_facts(): + # a list of facts descriptions to go in table header + headers = [] + # a list of inventory fact names + fact_names = [] + + # load the list of items/facts we want in our inventory + try: + inv_facts = app.config['INVENTORY_FACTS'] + except KeyError: + inv_facts = [('Hostname', 'fqdn'), + ('IP Address', 'ipaddress'), + ('OS', 'lsbdistdescription'), + ('Architecture', 'hardwaremodel'), + ('Kernel Version', 'kernelrelease')] + + # generate a list of descriptions and a list of fact names + # from the list of tuples inv_facts. + for desc, name in inv_facts: + headers.append(desc) + fact_names.append(name) + + return headers, fact_names + + +@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//inventory') +def inventory(env): + """Fetch all (active) nodes from PuppetDB and stream a table displaying + those nodes along with a set of facts about them. + + :param env: Search for facts in this environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + headers, fact_names = inventory_facts() + + return render_template( + 'inventory.html', + envs=envs, + current_env=env, + fact_headers=headers) + + +@app.route('/inventory/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//inventory/json') +def inventory_ajax(env): + """Backend endpoint for inventory table""" + draw = int(request.args.get('draw', 0)) + + envs = environments() + check_env(env, envs) + headers, fact_names = inventory_facts() + + query = AndOperator() + fact_query = OrOperator() + fact_query.add([EqualsOperator("name", name) for name in fact_names]) + query.add(fact_query) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + facts = puppetboard.facts.facts(query=query) + + fact_data = {} + for fact in facts: + if fact.node not in fact_data: + fact_data[fact.node] = {} + fact_data[fact.node][fact.name] = fact.value + + total = len(fact_data) + + return render_template( + 'inventory.json.tpl', + draw=draw, + total=total, + total_filtered=total, + fact_data=fact_data, + columns=fact_names) \ No newline at end of file diff --git a/puppetboard/metrics.py b/puppetboard/metrics.py new file mode 100644 index 000000000..73867c076 --- /dev/null +++ b/puppetboard/metrics.py @@ -0,0 +1,89 @@ +from urllib.parse import unquote + +from flask import render_template + +from puppetboard.app import app, check_env, puppetdb, metric_params +from puppetboard.core import environments +from puppetboard.utils import get_db_version, get_or_abort + + +@app.route('/metrics', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//metrics') +def metrics(env): + """Lists all available metrics that PuppetDB is aware of. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + if metric_version == 'v1': + mbeans = get_or_abort(puppetdb._query, 'mbean') + metrics = list(mbeans.keys()) + elif metric_version == 'v2': + # the list response is a dict in the format: + # { + # "domain1": { + # "property1": { + # ... + # } + # }, + # "domain2": { + # "property2": { + # ... + # } + # } + # } + # The MBean names are the combination of the domain and the properties + # with a ":" in between, example: + # domain1:property1 + # domain2:property2 + # reference: https://jolokia.org/reference/html/protocol.html#list + metrics_domains = get_or_abort(puppetdb.metric) + metrics = [] + # get all of the domains + for domain in list(metrics_domains.keys()): + # iterate over all of the properties in this domain + properties = list(metrics_domains[domain].keys()) + for prop in properties: + # combine the current domain and each property with + # a ":" in between + metrics.append(domain + ':' + prop) + else: + raise ValueError("Unknown metric version {} for database version {}" + .format(metric_version, db_version)) + + return render_template('metrics.html', + metrics=sorted(metrics), + envs=envs, + current_env=env) + + +@app.route('/metric/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//metric/') +def metric(env, metric): + """Lists all information about the metric of the given name. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + name = unquote(metric) + metric = get_or_abort(puppetdb.metric, metric, version=metric_version) + return render_template( + 'metric.html', + name=name, + metric=sorted(metric.items()), + envs=envs, + current_env=env) \ No newline at end of file diff --git a/puppetboard/nodes.py b/puppetboard/nodes.py new file mode 100644 index 000000000..a10663275 --- /dev/null +++ b/puppetboard/nodes.py @@ -0,0 +1,76 @@ +from datetime import datetime, timedelta + +from flask import request, Response, stream_with_context +from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, OrOperator, NullOperator, LessEqualOperator + +from puppetboard.app import app, check_env, puppetdb +from puppetboard.core import environments +from puppetboard.utils import yield_or_stop + + +@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//nodes') +def nodes(env): + """Fetch all (active) nodes from PuppetDB and stream a table displaying + those nodes. + + Downside of the streaming aproach is that since we've already sent our + headers we can't abort the request if we detect an error. Because of this + we'll end up with an empty table instead because of how yield_or_stop + works. Once pagination is in place we can change this but we'll need to + provide a search feature instead. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` + """ + envs = environments() + status_arg = request.args.get('status', '') + check_env(env, envs) + + query = AndOperator() + + if env != '*': + query.add(EqualsOperator("catalog_environment", env)) + + if status_arg in ['failed', 'changed', 'unchanged']: + query.add(EqualsOperator('latest_report_status', status_arg)) + elif status_arg == 'unreported': + unreported = datetime.utcnow() + unreported = (unreported - + timedelta(hours=app.config['UNRESPONSIVE_HOURS'])) + unreported = unreported.replace(microsecond=0).isoformat() + + unrep_query = OrOperator() + unrep_query.add(NullOperator('report_timestamp', True)) + unrep_query.add(LessEqualOperator('report_timestamp', unreported)) + + query.add(unrep_query) + + if len(query.operations) == 0: + query = None + + nodelist = puppetdb.nodes( + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) + nodes = [] + for node in yield_or_stop(nodelist): + if status_arg: + if node.status == status_arg: + nodes.append(node) + else: + nodes.append(node) + return Response(stream_with_context( + stream_template('nodes.html', + nodes=nodes, + envs=envs, + current_env=env))) + + +def stream_template(template_name, **context): + app.update_template_context(context) + t = app.jinja_env.get_template(template_name) + rv = t.stream(context) + rv.enable_buffering(5) + return rv \ No newline at end of file diff --git a/puppetboard/query.py b/puppetboard/query.py new file mode 100644 index 000000000..d950d6440 --- /dev/null +++ b/puppetboard/query.py @@ -0,0 +1,96 @@ +from collections import OrderedDict + +import logging +from flask import session, render_template +from flask_wtf import FlaskForm +from werkzeug.exceptions import abort +from wtforms import (BooleanField, SelectField, TextAreaField, validators) + +from puppetboard.app import app, check_env, puppetdb +from puppetboard.core import environments +from puppetboard.utils import get_or_abort + +numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) +logging.basicConfig(level=numeric_level) +log = logging.getLogger(__name__) + + +QUERY_ENDPOINTS = OrderedDict([ + # PuppetDB API endpoint, Form name + ('pql', 'PQL'), + ('nodes', 'Nodes'), + ('resources', 'Resources'), + ('facts', 'Facts'), + ('factsets', 'Fact Sets'), + ('fact-paths', 'Fact Paths'), + ('fact-contents', 'Fact Contents'), + ('reports', 'Reports'), + ('events', 'Events'), + ('catalogs', 'Catalogs'), + ('edges', 'Edges'), + ('environments', 'Environments'), +]) +ENABLED_QUERY_ENDPOINTS = app.config.get('ENABLED_QUERY_ENDPOINTS', list(QUERY_ENDPOINTS.keys())) + + +class QueryForm(FlaskForm): + """The form used to allow freeform queries to be executed against + PuppetDB.""" + query = TextAreaField('Query', [validators.DataRequired( + message='A query is required.')]) + endpoints = SelectField('API endpoint', choices=[ + (key, value) for key, value in QUERY_ENDPOINTS.items() + if key in ENABLED_QUERY_ENDPOINTS], default='pql') + rawjson = BooleanField('Raw JSON') + + +@app.route('/query', methods=('GET', 'POST'), + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//query', methods=('GET', 'POST')) +def query(env): + """Allows to execute raw, user created querries against PuppetDB. This is + currently highly experimental and explodes in interesting ways since none + of the possible exceptions are being handled just yet. This will return + the JSON of the response or a message telling you what whent wrong / + why nothing was returned. + + :param env: Serves no purpose for the query data but is required for the + select field in the environment block + :type env: :obj:`string` + """ + if not app.config['ENABLE_QUERY']: + log.warn('Access to query interface disabled by administrator.') + abort(403) + + envs = environments() + check_env(env, envs) + + form = QueryForm(meta={ + 'csrf_secret': app.config['SECRET_KEY'], + 'csrf_context': session}) + if form.validate_on_submit(): + if form.endpoints.data not in ENABLED_QUERY_ENDPOINTS: + log.warn('Access to query endpoint %s disabled by administrator.', + form.endpoints.data) + abort(403) + + if form.endpoints.data == 'pql': + query = form.query.data + elif form.query.data[0] == '[': + query = form.query.data + else: + query = '[{0}]'.format(form.query.data) + + result = get_or_abort( + puppetdb._query, + form.endpoints.data, + query=query) + return render_template('query.html', + form=form, + result=result, + envs=envs, + current_env=env) + return render_template('query.html', + form=form, + envs=envs, + current_env=env) \ No newline at end of file diff --git a/puppetboard/radiator.py b/puppetboard/radiator.py new file mode 100644 index 000000000..1602e8593 --- /dev/null +++ b/puppetboard/radiator.py @@ -0,0 +1,106 @@ +from flask import request, jsonify, render_template +from pypuppetdb.QueryBuilder import AndOperator, ExtractOperator, EqualsOperator, FunctionOperator + +import puppetboard.metrics +import puppetboard.nodes +from puppetboard.app import app, check_env, puppetdb, metric_params +from puppetboard.core import environments +from puppetboard.utils import get_db_version, get_or_abort + + +@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//radiator') +def radiator(env): + """This view generates a simplified monitoring page + akin to the radiator view in puppet dashboard + """ + envs = environments() + check_env(env, envs) + + if env == '*': + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + query = None + metrics = get_or_abort( + puppetboard.metrics.metric, + 'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type, + version=metric_version) + num_nodes = metrics['Value'] + else: + query = AndOperator() + metric_query = ExtractOperator() + + query.add(EqualsOperator("catalog_environment", env)) + metric_query.add_field(FunctionOperator('count')) + metric_query.add_query(query) + + metrics = get_or_abort( + puppetdb._query, + 'nodes', + query=metric_query) + num_nodes = metrics[0]['count'] + + nodes = puppetboard.nodes.nodes( + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True + ) + + stats = { + 'changed_percent': 0, + 'changed': 0, + 'failed_percent': 0, + 'failed': 0, + 'noop_percent': 0, + 'noop': 0, + 'skipped_percent': 0, + 'skipped': 0, + 'unchanged_percent': 0, + 'unchanged': 0, + 'unreported_percent': 0, + 'unreported': 0, + } + + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + elif node.status == 'skipped': + stats['skipped'] += 1 + else: + stats['unchanged'] += 1 + + try: + stats['changed_percent'] = int(100 * (stats['changed'] / + float(num_nodes))) + stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) + stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) + stats['skipped_percent'] = int(100 * (stats['skipped'] / + float(num_nodes))) + stats['unchanged_percent'] = int(100 * (stats['unchanged'] / + float(num_nodes))) + stats['unreported_percent'] = int(100 * (stats['unreported'] / + float(num_nodes))) + except ZeroDivisionError: + stats['changed_percent'] = 0 + stats['failed_percent'] = 0 + stats['noop_percent'] = 0 + stats['skipped_percent'] = 0 + stats['unchanged_percent'] = 0 + stats['unreported_percent'] = 0 + + if ('Accept' in request.headers and + request.headers["Accept"] == 'application/json'): + return jsonify(**stats) + + return render_template( + 'radiator.html', + stats=stats, + total=num_nodes + ) \ No newline at end of file diff --git a/puppetboard/reports.py b/puppetboard/reports.py new file mode 100644 index 000000000..ef4e59fe0 --- /dev/null +++ b/puppetboard/reports.py @@ -0,0 +1,223 @@ +import json + +import commonmark +from flask import render_template, request, abort +from itertools import tee +from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, OrOperator, RegexOperator, GreaterEqualOperator, \ + LessEqualOperator + +import puppetboard.metrics +from puppetboard.app import app, check_env, puppetdb +from puppetboard.core import environments +from puppetboard.utils import get_or_abort, yield_or_stop + + +@app.route('/reports', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node_name': None}) +@app.route('//reports', defaults={'node_name': None}) +@app.route('/reports/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//reports/') +def reports(env, node_name): + """Query and Return JSON data to reports Jquery datatable + + :param env: Search for all reports in this environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + return render_template( + 'reports.html', + envs=envs, + current_env=env, + node_name=node_name, + columns=REPORTS_COLUMNS) + + +@app.route('/reports/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node_name': None}) +@app.route('//reports/json', defaults={'node_name': None}) +@app.route('/reports//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//reports//json') +def reports_ajax(env, node_name): + """Query and Return JSON data to reports Jquery datatable + + :param env: Search for all reports in this environment + :type env: :obj:`string` + """ + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) + paging_args = {'limit': length, 'offset': start} + search_arg = request.args.get('search[value]') + order_column = int(request.args.get('order[0][column]', 0)) + order_filter = REPORTS_COLUMNS[order_column].get( + 'filter', REPORTS_COLUMNS[order_column]['attr']) + order_dir = request.args.get('order[0][dir]', 'desc') + order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) + status_args = request.args.get('columns[1][search][value]', '').split('|') + date_args = request.args.get('columns[0][search][value]', '') + max_col = len(REPORTS_COLUMNS) + for i in range(len(REPORTS_COLUMNS)): + if request.args.get("columns[%s][data]" % i, None): + max_col = i + 1 + + envs = environments() + check_env(env, envs) + reports_query = AndOperator() + + if env != '*': + reports_query.add(EqualsOperator("environment", env)) + + if node_name: + reports_query.add(EqualsOperator("certname", node_name)) + + if search_arg: + search_query = OrOperator() + search_query.add(RegexOperator("certname", r"%s" % search_arg)) + search_query.add(RegexOperator("puppet_version", r"%s" % search_arg)) + search_query.add(RegexOperator( + "configuration_version", r"%s" % search_arg)) + reports_query.add(search_query) + + if date_args: + dates = json.loads(date_args) + + if len(dates) > 0: + date_query = AndOperator() + + if 'min' in dates: + date_query.add(GreaterEqualOperator('end_time', dates['min'])) + + if 'max' in dates: + date_query.add(LessEqualOperator('end_time', dates['max'])) + + reports_query.add(date_query) + + status_query = OrOperator() + for status_arg in status_args: + if status_arg in ['failed', 'changed', 'unchanged']: + arg_query = AndOperator() + arg_query.add(EqualsOperator('status', status_arg)) + arg_query.add(EqualsOperator('noop', False)) + status_query.add(arg_query) + if status_arg == 'unchanged': + arg_query = AndOperator() + arg_query.add(EqualsOperator('noop', True)) + arg_query.add(EqualsOperator('noop_pending', False)) + status_query.add(arg_query) + elif status_arg == 'noop': + arg_query = AndOperator() + arg_query.add(EqualsOperator('noop', True)) + arg_query.add(EqualsOperator('noop_pending', True)) + status_query.add(arg_query) + + if len(status_query.operations) == 0: + if len(reports_query.operations) == 0: + reports_query = None + else: + reports_query.add(status_query) + + if status_args[0] != 'none': + reports = get_or_abort( + puppetdb.reports, + query=reports_query, + order_by=order_args, + include_total=True, + **paging_args) + reports, reports_events = tee(reports) + total = None + else: + reports = [] + reports_events = [] + total = 0 + + # Convert metrics to relational dict + metrics = {} + for report in reports_events: + if total is None: + total = puppetdb.total + + metrics[report.hash_] = {} + for m in puppetboard.metrics.metrics: + if m['category'] not in metrics[report.hash_]: + metrics[report.hash_][m['category']] = {} + metrics[report.hash_][m['category']][m['name']] = m['value'] + + if total is None: + total = 0 + + return render_template( + 'reports.json.tpl', + draw=draw, + total=total, + total_filtered=total, + reports=reports, + metrics=metrics, + envs=envs, + current_env=env, + columns=REPORTS_COLUMNS[:max_col]) + + +@app.route('/report//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//report//') +def report(env, node_name, report_id): + """Displays a single report including all the events associated with that + report and their status. + + The report_id may be the puppetdb's report hash or the + configuration_version. This allows for better integration + into puppet-hipchat. + + :param env: Search for reports in this environment + :type env: :obj:`string` + :param node_name: Find the reports whose certname match this value + :type node_name: :obj:`string` + :param report_id: The hash or the configuration_version of the desired + report + :type report_id: :obj:`string` + """ + envs = environments() + check_env(env, envs) + query = AndOperator() + report_id_query = OrOperator() + + report_id_query.add(EqualsOperator("hash", report_id)) + report_id_query.add(EqualsOperator("configuration_version", report_id)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + query.add(EqualsOperator("certname", node_name)) + query.add(report_id_query) + + reports = puppetdb.reports(query=query) + + try: + report = next(reports) + except StopIteration: + abort(404) + + report.version = commonmark.commonmark(report.version) + + return render_template( + 'report.html', + report=report, + events=yield_or_stop(report.events()), + logs=report.logs, + metrics=puppetboard.metrics.metrics, + envs=envs, + current_env=env) + + +REPORTS_COLUMNS = [ + {'attr': 'end', 'filter': 'end_time', 'name': 'End time', 'type': 'datetime'}, + {'attr': 'status', 'name': 'Status', 'type': 'status'}, + {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, + {'attr': 'version', 'filter': 'configuration_version', 'name': 'Configuration version'}, + {'attr': 'agent_version', 'filter': 'puppet_version', 'name': 'Agent version'}, +] \ No newline at end of file diff --git a/puppetboard/utils.py b/puppetboard/utils.py index b68455117..510aead70 100644 --- a/puppetboard/utils.py +++ b/puppetboard/utils.py @@ -8,10 +8,14 @@ from pypuppetdb.errors import EmptyResponseError from requests.exceptions import ConnectionError, HTTPError +from puppetboard.app import app +numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) +logging.basicConfig(level=numeric_level) log = logging.getLogger(__name__) + @contextfunction def url_static_offline(context, value): request_parts = os.path.split(os.path.dirname(context.name)) From bc0ddf5e1e07db2077ea84977c6d0361d00ae394 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 11 Oct 2021 09:44:20 +0200 Subject: [PATCH 10/10] Revert "Refactor: split each tab into separate file" This reverts commit 5ff84347155b5d7781bed69a43527319f29bef55. --- puppetboard/app.py | 1130 +++++++++++++++++++++++++++++++++++++- puppetboard/catalogs.py | 165 ------ puppetboard/facts.py | 198 ------- puppetboard/forms.py | 36 ++ puppetboard/index.py | 170 ------ puppetboard/inventory.py | 89 --- puppetboard/metrics.py | 89 --- puppetboard/nodes.py | 76 --- puppetboard/query.py | 96 ---- puppetboard/radiator.py | 106 ---- puppetboard/reports.py | 223 -------- puppetboard/utils.py | 4 - 12 files changed, 1157 insertions(+), 1225 deletions(-) delete mode 100644 puppetboard/catalogs.py delete mode 100644 puppetboard/facts.py create mode 100644 puppetboard/forms.py delete mode 100644 puppetboard/index.py delete mode 100644 puppetboard/inventory.py delete mode 100644 puppetboard/metrics.py delete mode 100644 puppetboard/nodes.py delete mode 100644 puppetboard/query.py delete mode 100644 puppetboard/radiator.py delete mode 100644 puppetboard/reports.py diff --git a/puppetboard/app.py b/puppetboard/app.py index 7054dc973..1b9a6e8f5 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -1,21 +1,59 @@ -import logging -from datetime import datetime +from __future__ import absolute_import +from __future__ import unicode_literals +from urllib.parse import unquote, unquote_plus, quote_plus +from datetime import datetime, timedelta +from itertools import tee +from distutils.util import strtobool from flask import ( - render_template, abort, Response, request, jsonify + render_template, abort, url_for, + Response, stream_with_context, request, session, jsonify ) -from pypuppetdb.QueryBuilder import (AndOperator, - EqualsOperator) -from puppetboard.core import get_app, get_puppetdb, environments +import logging +import json + +from pypuppetdb.QueryBuilder import (ExtractOperator, AndOperator, + EqualsOperator, FunctionOperator, + NullOperator, OrOperator, + LessEqualOperator, RegexOperator, + GreaterEqualOperator) + +from pypuppetdb.types import Node +from pypuppetdb.utils import json_to_datetime + +from puppetboard.forms import ENABLED_QUERY_ENDPOINTS, QueryForm +from puppetboard.utils import (get_or_abort, yield_or_stop, + get_db_version, is_bool) from puppetboard.dailychart import get_daily_reports_chart -from puppetboard.reports import REPORTS_COLUMNS -from puppetboard.utils import (get_or_abort) + +import commonmark + +from puppetboard.core import get_app, get_puppetdb, environments + from puppetboard.version import __version__ -app = get_app() +REPORTS_COLUMNS = [ + {'attr': 'end', 'filter': 'end_time', + 'name': 'End time', 'type': 'datetime'}, + {'attr': 'status', 'name': 'Status', 'type': 'status'}, + {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, + {'attr': 'version', 'filter': 'configuration_version', + 'name': 'Configuration version'}, + {'attr': 'agent_version', 'filter': 'puppet_version', + 'name': 'Agent version'}, +] + +CATALOGS_COLUMNS = [ + {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, + {'attr': 'catalog_timestamp', 'name': 'Compile Time'}, + {'attr': 'form', 'name': 'Compare'}, +] +app = get_app() +graph_facts = app.config['GRAPH_FACTS'] numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) + logging.basicConfig(level=numeric_level) log = logging.getLogger(__name__) @@ -49,6 +87,14 @@ def version(): return __version__ +def stream_template(template_name, **context): + app.update_template_context(context) + t = app.jinja_env.get_template(template_name) + rv = t.stream(context) + rv.enable_buffering(5) + return rv + + def check_env(env, envs): if env != '*' and env not in envs: abort(404) @@ -83,6 +129,307 @@ def now(format='%m/%d/%Y %H:%M:%S'): return dict(now=now) +@app.route('/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//') +def index(env): + """This view generates the index page and displays a set of metrics and + latest reports on nodes fetched from PuppetDB. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + metrics = { + 'num_nodes': 0, + 'num_resources': 0, + 'avg_resources_node': 0, + } + nodes_overview = [] + stats = { + 'changed': 0, + 'unchanged': 0, + 'failed': 0, + 'unreported': 0, + 'noop': 0 + } + + nodes = [] + + node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] + resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] + if env == '*': + query = app.config['OVERVIEW_FILTER'] + + prefix = 'puppetlabs.puppetdb.population' + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + num_nodes = get_or_abort( + puppetdb.metric, + "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), + version=metric_version) + + if resource_stats_enabled: + num_resources = get_or_abort( + puppetdb.metric, + "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), + version=metric_version) + + metrics['num_resources'] = num_resources['Value'] + try: + # Compute our own average because avg_resources_node['Value'] + # returns a string of the format "num_resources/num_nodes" + # example: "1234/9" instead of doing the division itself. + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources['Value'] / num_nodes['Value'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + else: + query = AndOperator() + query.add(EqualsOperator('catalog_environment', env)) + + num_nodes_query = ExtractOperator() + num_nodes_query.add_field(FunctionOperator('count')) + num_nodes_query.add_query(query) + + if app.config['OVERVIEW_FILTER'] is not None: + query.add(app.config['OVERVIEW_FILTER']) + + num_nodes = get_or_abort( + puppetdb._query, + 'nodes', + query=num_nodes_query) + + metrics['num_nodes'] = num_nodes[0]['count'] + + if resource_stats_enabled: + num_resources_query = ExtractOperator() + num_resources_query.add_field(FunctionOperator('count')) + num_resources_query.add_query(EqualsOperator("environment", env)) + + num_resources = get_or_abort( + puppetdb._query, + 'resources', + query=num_resources_query) + + metrics['num_resources'] = num_resources[0]['count'] + try: + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources[0]['count'] / num_nodes[0]['count'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + + if node_status_detail_enabled: + nodes = get_or_abort(puppetdb.nodes, + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) + + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + else: + stats['unchanged'] += 1 + + if node_status_detail_enabled: + if node.status != 'unchanged': + nodes_overview.append(node) + + return render_template( + 'index.html', + metrics=metrics, + nodes=nodes_overview, + stats=stats, + envs=envs, + current_env=env + ) + + +def get_node_status_summary(inner_query): + node_status_query = ExtractOperator() + node_status_query.add_field('certname') + node_status_query.add_field('report_timestamp') + node_status_query.add_field('latest_report_status') + node_status_query.add_query(inner_query) + + node_status = get_or_abort( + puppetdb._query, + 'nodes', + query=node_status_query + ) + + now = datetime.utcnow() + node_infos = [] + for node_state in node_status: + last_report = json_to_datetime(node_state['report_timestamp']) + last_report = last_report.replace(tzinfo=None) + unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) + certname = node_state['certname'] + + node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, + status_report=node_state['latest_report_status'])) + + return node_infos + + +@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//nodes') +def nodes(env): + """Fetch all (active) nodes from PuppetDB and stream a table displaying + those nodes. + + Downside of the streaming aproach is that since we've already sent our + headers we can't abort the request if we detect an error. Because of this + we'll end up with an empty table instead because of how yield_or_stop + works. Once pagination is in place we can change this but we'll need to + provide a search feature instead. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` + """ + envs = environments() + status_arg = request.args.get('status', '') + check_env(env, envs) + + query = AndOperator() + + if env != '*': + query.add(EqualsOperator("catalog_environment", env)) + + if status_arg in ['failed', 'changed', 'unchanged']: + query.add(EqualsOperator('latest_report_status', status_arg)) + elif status_arg == 'unreported': + unreported = datetime.utcnow() + unreported = (unreported - + timedelta(hours=app.config['UNRESPONSIVE_HOURS'])) + unreported = unreported.replace(microsecond=0).isoformat() + + unrep_query = OrOperator() + unrep_query.add(NullOperator('report_timestamp', True)) + unrep_query.add(LessEqualOperator('report_timestamp', unreported)) + + query.add(unrep_query) + + if len(query.operations) == 0: + query = None + + nodelist = puppetdb.nodes( + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) + nodes = [] + for node in yield_or_stop(nodelist): + if status_arg: + if node.status == status_arg: + nodes.append(node) + else: + nodes.append(node) + return Response(stream_with_context( + stream_template('nodes.html', + nodes=nodes, + envs=envs, + current_env=env))) + + +def inventory_facts(): + # a list of facts descriptions to go in table header + headers = [] + # a list of inventory fact names + fact_names = [] + + # load the list of items/facts we want in our inventory + try: + inv_facts = app.config['INVENTORY_FACTS'] + except KeyError: + inv_facts = [('Hostname', 'fqdn'), + ('IP Address', 'ipaddress'), + ('OS', 'lsbdistdescription'), + ('Architecture', 'hardwaremodel'), + ('Kernel Version', 'kernelrelease')] + + # generate a list of descriptions and a list of fact names + # from the list of tuples inv_facts. + for desc, name in inv_facts: + headers.append(desc) + fact_names.append(name) + + return headers, fact_names + + +@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//inventory') +def inventory(env): + """Fetch all (active) nodes from PuppetDB and stream a table displaying + those nodes along with a set of facts about them. + + :param env: Search for facts in this environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + headers, fact_names = inventory_facts() + + return render_template( + 'inventory.html', + envs=envs, + current_env=env, + fact_headers=headers) + + +@app.route('/inventory/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//inventory/json') +def inventory_ajax(env): + """Backend endpoint for inventory table""" + draw = int(request.args.get('draw', 0)) + + envs = environments() + check_env(env, envs) + headers, fact_names = inventory_facts() + + query = AndOperator() + fact_query = OrOperator() + fact_query.add([EqualsOperator("name", name) for name in fact_names]) + query.add(fact_query) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + facts = puppetdb.facts(query=query) + + fact_data = {} + for fact in facts: + if fact.node not in fact_data: + fact_data[fact.node] = {} + fact_data[fact.node][fact.name] = fact.value + + total = len(fact_data) + + return render_template( + 'inventory.json.tpl', + draw=draw, + total=total, + total_filtered=total, + fact_data=fact_data, + columns=fact_names) + + @app.route('/node/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//node/') @@ -113,6 +460,771 @@ def node(env, node_name): columns=REPORTS_COLUMNS[:2]) +@app.route('/reports', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node_name': None}) +@app.route('//reports', defaults={'node_name': None}) +@app.route('/reports/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//reports/') +def reports(env, node_name): + """Query and Return JSON data to reports Jquery datatable + + :param env: Search for all reports in this environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + return render_template( + 'reports.html', + envs=envs, + current_env=env, + node_name=node_name, + columns=REPORTS_COLUMNS) + + +@app.route('/reports/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node_name': None}) +@app.route('//reports/json', defaults={'node_name': None}) +@app.route('/reports//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//reports//json') +def reports_ajax(env, node_name): + """Query and Return JSON data to reports Jquery datatable + + :param env: Search for all reports in this environment + :type env: :obj:`string` + """ + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) + paging_args = {'limit': length, 'offset': start} + search_arg = request.args.get('search[value]') + order_column = int(request.args.get('order[0][column]', 0)) + order_filter = REPORTS_COLUMNS[order_column].get( + 'filter', REPORTS_COLUMNS[order_column]['attr']) + order_dir = request.args.get('order[0][dir]', 'desc') + order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) + status_args = request.args.get('columns[1][search][value]', '').split('|') + date_args = request.args.get('columns[0][search][value]', '') + max_col = len(REPORTS_COLUMNS) + for i in range(len(REPORTS_COLUMNS)): + if request.args.get("columns[%s][data]" % i, None): + max_col = i + 1 + + envs = environments() + check_env(env, envs) + reports_query = AndOperator() + + if env != '*': + reports_query.add(EqualsOperator("environment", env)) + + if node_name: + reports_query.add(EqualsOperator("certname", node_name)) + + if search_arg: + search_query = OrOperator() + search_query.add(RegexOperator("certname", r"%s" % search_arg)) + search_query.add(RegexOperator("puppet_version", r"%s" % search_arg)) + search_query.add(RegexOperator( + "configuration_version", r"%s" % search_arg)) + reports_query.add(search_query) + + if date_args: + dates = json.loads(date_args) + + if len(dates) > 0: + date_query = AndOperator() + + if 'min' in dates: + date_query.add(GreaterEqualOperator('end_time', dates['min'])) + + if 'max' in dates: + date_query.add(LessEqualOperator('end_time', dates['max'])) + + reports_query.add(date_query) + + status_query = OrOperator() + for status_arg in status_args: + if status_arg in ['failed', 'changed', 'unchanged']: + arg_query = AndOperator() + arg_query.add(EqualsOperator('status', status_arg)) + arg_query.add(EqualsOperator('noop', False)) + status_query.add(arg_query) + if status_arg == 'unchanged': + arg_query = AndOperator() + arg_query.add(EqualsOperator('noop', True)) + arg_query.add(EqualsOperator('noop_pending', False)) + status_query.add(arg_query) + elif status_arg == 'noop': + arg_query = AndOperator() + arg_query.add(EqualsOperator('noop', True)) + arg_query.add(EqualsOperator('noop_pending', True)) + status_query.add(arg_query) + + if len(status_query.operations) == 0: + if len(reports_query.operations) == 0: + reports_query = None + else: + reports_query.add(status_query) + + if status_args[0] != 'none': + reports = get_or_abort( + puppetdb.reports, + query=reports_query, + order_by=order_args, + include_total=True, + **paging_args) + reports, reports_events = tee(reports) + total = None + else: + reports = [] + reports_events = [] + total = 0 + + # Convert metrics to relational dict + metrics = {} + for report in reports_events: + if total is None: + total = puppetdb.total + + metrics[report.hash_] = {} + for m in report.metrics: + if m['category'] not in metrics[report.hash_]: + metrics[report.hash_][m['category']] = {} + metrics[report.hash_][m['category']][m['name']] = m['value'] + + if total is None: + total = 0 + + return render_template( + 'reports.json.tpl', + draw=draw, + total=total, + total_filtered=total, + reports=reports, + metrics=metrics, + envs=envs, + current_env=env, + columns=REPORTS_COLUMNS[:max_col]) + + +@app.route('/report//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//report//') +def report(env, node_name, report_id): + """Displays a single report including all the events associated with that + report and their status. + + The report_id may be the puppetdb's report hash or the + configuration_version. This allows for better integration + into puppet-hipchat. + + :param env: Search for reports in this environment + :type env: :obj:`string` + :param node_name: Find the reports whose certname match this value + :type node_name: :obj:`string` + :param report_id: The hash or the configuration_version of the desired + report + :type report_id: :obj:`string` + """ + envs = environments() + check_env(env, envs) + query = AndOperator() + report_id_query = OrOperator() + + report_id_query.add(EqualsOperator("hash", report_id)) + report_id_query.add(EqualsOperator("configuration_version", report_id)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + query.add(EqualsOperator("certname", node_name)) + query.add(report_id_query) + + reports = puppetdb.reports(query=query) + + try: + report = next(reports) + except StopIteration: + abort(404) + + report.version = commonmark.commonmark(report.version) + + return render_template( + 'report.html', + report=report, + events=yield_or_stop(report.events()), + logs=report.logs, + metrics=report.metrics, + envs=envs, + current_env=env) + + +@app.route('/facts', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//facts') +def facts(env): + """Displays an alphabetical list of all facts currently known to + PuppetDB. + + :param env: Serves no purpose for this function, only for consistency's + sake + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + facts = get_or_abort(puppetdb.fact_names) + + facts_columns = [[]] + letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size + count = 0 + for fact in facts: + count += 1 + + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset + letter = fact[0].upper() + letter_list = [] + + letter_list.append(fact) + facts_columns[-1].append(letter_list) + + return render_template('facts.html', + facts_columns=facts_columns, + envs=envs, + current_env=env) + + +@app.route('/fact/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) +@app.route('//fact/', defaults={'value': None}) +@app.route('/fact//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//fact//') +def fact(env, fact, value): + """Fetches the specific fact(/value) from PuppetDB and displays per + node for which this fact is known. + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Find all facts with this value + :type value: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + render_graph = False + if fact in graph_facts and not value: + render_graph = True + + value_safe = value + if value is not None: + value_safe = unquote_plus(value) + + return render_template( + 'fact.html', + fact=fact, + value=value, + value_safe=value_safe, + render_graph=render_graph, + envs=envs, + current_env=env) + + +@app.route('/fact//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node': None, 'value': None}) +@app.route('//fact//json', defaults={'node': None, 'value': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('//fact///json', defaults={'node': None}) +@app.route('/node//facts/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'fact': None, 'value': None}) +@app.route('//node//facts/json', + defaults={'fact': None, 'value': None}) +def fact_ajax(env, node, fact, value): + """Fetches the specific facts matching (node/fact/value) from PuppetDB and + return a JSON table + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param node: Find all facts for this node + :type node: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Filter facts whose value is equal to this + :type value: :obj:`string` + """ + draw = int(request.args.get('draw', 0)) + + envs = environments() + check_env(env, envs) + + render_graph = False + if fact in graph_facts and not value and not node: + render_graph = True + + query = AndOperator() + if node: + query.add(EqualsOperator("certname", node)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + if len(query.operations) == 0: + query = None + + # Generator needs to be converted (graph / total) + try: + value = int(value) + except ValueError: + if value is not None and query is not None: + if is_bool(value): + query.add(EqualsOperator('value', bool(strtobool(value)))) + else: + query.add(EqualsOperator('value', unquote_plus(value))) + except TypeError: + pass + + facts = [f for f in get_or_abort( + puppetdb.facts, + name=fact, + query=query)] + + total = len(facts) + + counts = {} + json = { + 'draw': draw, + 'recordsTotal': total, + 'recordsFiltered': total, + 'data': []} + + for fact_h in facts: + line = [] + if not fact: + line.append(fact_h.name) + if not node: + line.append('{1}'.format( + url_for('node', env=env, node_name=fact_h.node), + fact_h.node)) + if not value: + fact_value = fact_h.value + if isinstance(fact_value, str): + fact_value = quote_plus(fact_h.value) + + line.append('{1}'.format( + url_for( + 'fact', env=env, fact=fact_h.name, value=fact_value), + fact_h.value)) + + json['data'].append(line) + + if render_graph: + if fact_h.value not in counts: + counts[fact_h.value] = 0 + counts[fact_h.value] += 1 + + if render_graph: + json['chart'] = [ + {"label": "{0}".format(k).replace('\n', ' '), + "value": counts[k]} + for k in sorted(counts, key=lambda k: counts[k], reverse=True)] + + return jsonify(json) + + +@app.route('/query', methods=('GET', 'POST'), + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//query', methods=('GET', 'POST')) +def query(env): + """Allows to execute raw, user created querries against PuppetDB. This is + currently highly experimental and explodes in interesting ways since none + of the possible exceptions are being handled just yet. This will return + the JSON of the response or a message telling you what whent wrong / + why nothing was returned. + + :param env: Serves no purpose for the query data but is required for the + select field in the environment block + :type env: :obj:`string` + """ + if not app.config['ENABLE_QUERY']: + log.warn('Access to query interface disabled by administrator.') + abort(403) + + envs = environments() + check_env(env, envs) + + form = QueryForm(meta={ + 'csrf_secret': app.config['SECRET_KEY'], + 'csrf_context': session}) + if form.validate_on_submit(): + if form.endpoints.data not in ENABLED_QUERY_ENDPOINTS: + log.warn('Access to query endpoint %s disabled by administrator.', + form.endpoints.data) + abort(403) + + if form.endpoints.data == 'pql': + query = form.query.data + elif form.query.data[0] == '[': + query = form.query.data + else: + query = '[{0}]'.format(form.query.data) + + result = get_or_abort( + puppetdb._query, + form.endpoints.data, + query=query) + return render_template('query.html', + form=form, + result=result, + envs=envs, + current_env=env) + return render_template('query.html', + form=form, + envs=envs, + current_env=env) + + +@app.route('/metrics', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//metrics') +def metrics(env): + """Lists all available metrics that PuppetDB is aware of. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + if metric_version == 'v1': + mbeans = get_or_abort(puppetdb._query, 'mbean') + metrics = list(mbeans.keys()) + elif metric_version == 'v2': + # the list response is a dict in the format: + # { + # "domain1": { + # "property1": { + # ... + # } + # }, + # "domain2": { + # "property2": { + # ... + # } + # } + # } + # The MBean names are the combination of the domain and the properties + # with a ":" in between, example: + # domain1:property1 + # domain2:property2 + # reference: https://jolokia.org/reference/html/protocol.html#list + metrics_domains = get_or_abort(puppetdb.metric) + metrics = [] + # get all of the domains + for domain in list(metrics_domains.keys()): + # iterate over all of the properties in this domain + properties = list(metrics_domains[domain].keys()) + for prop in properties: + # combine the current domain and each property with + # a ":" in between + metrics.append(domain + ':' + prop) + else: + raise ValueError("Unknown metric version {} for database version {}" + .format(metric_version, db_version)) + + return render_template('metrics.html', + metrics=sorted(metrics), + envs=envs, + current_env=env) + + +@app.route('/metric/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//metric/') +def metric(env, metric): + """Lists all information about the metric of the given name. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + name = unquote(metric) + metric = get_or_abort(puppetdb.metric, metric, version=metric_version) + return render_template( + 'metric.html', + name=name, + metric=sorted(metric.items()), + envs=envs, + current_env=env) + + +@app.route('/catalogs', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'compare': None}) +@app.route('//catalogs', defaults={'compare': None}) +@app.route('/catalogs/compare/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare/') +def catalogs(env, compare): + """Lists all nodes with a compiled catalog. + + :param env: Find the nodes with this catalog_environment value + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if not app.config['ENABLE_CATALOG']: + log.warning('Access to catalog interface disabled by administrator') + abort(403) + + return render_template( + 'catalogs.html', + compare=compare, + columns=CATALOGS_COLUMNS, + envs=envs, + current_env=env) + + +@app.route('/catalogs/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'compare': None}) +@app.route('//catalogs/json', defaults={'compare': None}) +@app.route('/catalogs/compare//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare//json') +def catalogs_ajax(env, compare): + """Server data to catalogs as JSON to Jquery datatables + """ + draw = int(request.args.get('draw', 0)) + start = int(request.args.get('start', 0)) + length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) + paging_args = {'limit': length, 'offset': start} + search_arg = request.args.get('search[value]') + order_column = int(request.args.get('order[0][column]', 0)) + order_filter = CATALOGS_COLUMNS[order_column].get( + 'filter', CATALOGS_COLUMNS[order_column]['attr']) + order_dir = request.args.get('order[0][dir]', 'asc') + order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) + + envs = environments() + check_env(env, envs) + + query = AndOperator() + if env != '*': + query.add(EqualsOperator("catalog_environment", env)) + if search_arg: + query.add(RegexOperator("certname", r"%s" % search_arg)) + query.add(NullOperator("catalog_timestamp", False)) + + nodes = get_or_abort(puppetdb.nodes, + query=query, + include_total=True, + order_by=order_args, + **paging_args) + + catalog_list = [] + total = None + for node in nodes: + if total is None: + total = puppetdb.total + + catalog_list.append({ + 'certname': node.name, + 'catalog_timestamp': node.catalog_timestamp, + 'form': compare, + }) + + if total is None: + total = 0 + + return render_template( + 'catalogs.json.tpl', + total=total, + total_filtered=total, + draw=draw, + columns=CATALOGS_COLUMNS, + catalogs=catalog_list, + envs=envs, + current_env=env) + + +@app.route('/catalog/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalog/') +def catalog_node(env, node_name): + """Fetches from PuppetDB the compiled catalog of a given node. + + :param env: Find the catalog with this environment value + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if app.config['ENABLE_CATALOG']: + catalog = get_or_abort(puppetdb.catalog, + node=node_name) + return render_template('catalog.html', + catalog=catalog, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + + +@app.route('/catalogs/compare/...', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//catalogs/compare/...') +def catalog_compare(env, compare, against): + """Compares the catalog of one node, parameter compare, with that of + with that of another node, parameter against. + + :param env: Ensure that the 2 catalogs are in the same environment + :type env: :obj:`string` + """ + envs = environments() + check_env(env, envs) + + if app.config['ENABLE_CATALOG']: + compare_cat = get_or_abort(puppetdb.catalog, + node=compare) + against_cat = get_or_abort(puppetdb.catalog, + node=against) + + return render_template('catalog_compare.html', + compare=compare_cat, + against=against_cat, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + + +@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//radiator') +def radiator(env): + """This view generates a simplified monitoring page + akin to the radiator view in puppet dashboard + """ + envs = environments() + check_env(env, envs) + + if env == '*': + db_version = get_db_version(puppetdb) + query_type, metric_version = metric_params(db_version) + + query = None + metrics = get_or_abort( + puppetdb.metric, + 'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type, + version=metric_version) + num_nodes = metrics['Value'] + else: + query = AndOperator() + metric_query = ExtractOperator() + + query.add(EqualsOperator("catalog_environment", env)) + metric_query.add_field(FunctionOperator('count')) + metric_query.add_query(query) + + metrics = get_or_abort( + puppetdb._query, + 'nodes', + query=metric_query) + num_nodes = metrics[0]['count'] + + nodes = puppetdb.nodes( + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True + ) + + stats = { + 'changed_percent': 0, + 'changed': 0, + 'failed_percent': 0, + 'failed': 0, + 'noop_percent': 0, + 'noop': 0, + 'skipped_percent': 0, + 'skipped': 0, + 'unchanged_percent': 0, + 'unchanged': 0, + 'unreported_percent': 0, + 'unreported': 0, + } + + for node in nodes: + if node.status == 'unreported': + stats['unreported'] += 1 + elif node.status == 'changed': + stats['changed'] += 1 + elif node.status == 'failed': + stats['failed'] += 1 + elif node.status == 'noop': + stats['noop'] += 1 + elif node.status == 'skipped': + stats['skipped'] += 1 + else: + stats['unchanged'] += 1 + + try: + stats['changed_percent'] = int(100 * (stats['changed'] / + float(num_nodes))) + stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) + stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) + stats['skipped_percent'] = int(100 * (stats['skipped'] / + float(num_nodes))) + stats['unchanged_percent'] = int(100 * (stats['unchanged'] / + float(num_nodes))) + stats['unreported_percent'] = int(100 * (stats['unreported'] / + float(num_nodes))) + except ZeroDivisionError: + stats['changed_percent'] = 0 + stats['failed_percent'] = 0 + stats['noop_percent'] = 0 + stats['skipped_percent'] = 0 + stats['unchanged_percent'] = 0 + stats['unreported_percent'] = 0 + + if ('Accept' in request.headers and + request.headers["Accept"] == 'application/json'): + return jsonify(**stats) + + return render_template( + 'radiator.html', + stats=stats, + total=num_nodes + ) + + @app.route('/daily_reports_chart.json', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//daily_reports_chart.json') diff --git a/puppetboard/catalogs.py b/puppetboard/catalogs.py deleted file mode 100644 index 3ebdb90ba..000000000 --- a/puppetboard/catalogs.py +++ /dev/null @@ -1,165 +0,0 @@ -import logging - -from flask import render_template, request -from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, RegexOperator, NullOperator -from werkzeug.exceptions import abort - -import puppetboard.nodes -from puppetboard.app import app, check_env, puppetdb -from puppetboard.core import environments -from puppetboard.utils import get_or_abort - -numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) -logging.basicConfig(level=numeric_level) -log = logging.getLogger(__name__) - - -@app.route('/catalogs', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'compare': None}) -@app.route('//catalogs', defaults={'compare': None}) -@app.route('/catalogs/compare/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare/') -def catalogs(env, compare): - """Lists all nodes with a compiled catalog. - - :param env: Find the nodes with this catalog_environment value - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if not app.config['ENABLE_CATALOG']: - log.warning('Access to catalog interface disabled by administrator') - abort(403) - - return render_template( - 'catalogs.html', - compare=compare, - columns=CATALOGS_COLUMNS, - envs=envs, - current_env=env) - - -@app.route('/catalogs/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'compare': None}) -@app.route('//catalogs/json', defaults={'compare': None}) -@app.route('/catalogs/compare//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare//json') -def catalogs_ajax(env, compare): - """Server data to catalogs as JSON to Jquery datatables - """ - draw = int(request.args.get('draw', 0)) - start = int(request.args.get('start', 0)) - length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) - paging_args = {'limit': length, 'offset': start} - search_arg = request.args.get('search[value]') - order_column = int(request.args.get('order[0][column]', 0)) - order_filter = CATALOGS_COLUMNS[order_column].get( - 'filter', CATALOGS_COLUMNS[order_column]['attr']) - order_dir = request.args.get('order[0][dir]', 'asc') - order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) - - envs = environments() - check_env(env, envs) - - query = AndOperator() - if env != '*': - query.add(EqualsOperator("catalog_environment", env)) - if search_arg: - query.add(RegexOperator("certname", r"%s" % search_arg)) - query.add(NullOperator("catalog_timestamp", False)) - - nodes = get_or_abort(puppetboard.nodes.nodes, - query=query, - include_total=True, - order_by=order_args, - **paging_args) - - catalog_list = [] - total = None - for node in nodes: - if total is None: - total = puppetdb.total - - catalog_list.append({ - 'certname': node.name, - 'catalog_timestamp': node.catalog_timestamp, - 'form': compare, - }) - - if total is None: - total = 0 - - return render_template( - 'catalogs.json.tpl', - total=total, - total_filtered=total, - draw=draw, - columns=CATALOGS_COLUMNS, - catalogs=catalog_list, - envs=envs, - current_env=env) - - -@app.route('/catalog/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalog/') -def catalog_node(env, node_name): - """Fetches from PuppetDB the compiled catalog of a given node. - - :param env: Find the catalog with this environment value - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if app.config['ENABLE_CATALOG']: - catalog = get_or_abort(puppetdb.catalog, - node=node_name) - return render_template('catalog.html', - catalog=catalog, - envs=envs, - current_env=env) - else: - log.warn('Access to catalog interface disabled by administrator') - abort(403) - - -@app.route('/catalogs/compare/...', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//catalogs/compare/...') -def catalog_compare(env, compare, against): - """Compares the catalog of one node, parameter compare, with that of - with that of another node, parameter against. - - :param env: Ensure that the 2 catalogs are in the same environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - if app.config['ENABLE_CATALOG']: - compare_cat = get_or_abort(puppetdb.catalog, - node=compare) - against_cat = get_or_abort(puppetdb.catalog, - node=against) - - return render_template('catalog_compare.html', - compare=compare_cat, - against=against_cat, - envs=envs, - current_env=env) - else: - log.warn('Access to catalog interface disabled by administrator') - abort(403) - - -CATALOGS_COLUMNS = [ - {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, - {'attr': 'catalog_timestamp', 'name': 'Compile Time'}, - {'attr': 'form', 'name': 'Compare'}, -] \ No newline at end of file diff --git a/puppetboard/facts.py b/puppetboard/facts.py deleted file mode 100644 index 0728dd9c2..000000000 --- a/puppetboard/facts.py +++ /dev/null @@ -1,198 +0,0 @@ -from distutils.util import strtobool -from urllib.parse import unquote_plus, quote_plus - -from flask import render_template, request, url_for, jsonify -from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator - -from puppetboard.app import app, check_env, puppetdb -from puppetboard.core import environments -from puppetboard.utils import get_or_abort, is_bool - -graph_facts = app.config['GRAPH_FACTS'] - - -@app.route('/facts', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//facts') -def facts(env): - """Displays an alphabetical list of all facts currently known to - PuppetDB. - - :param env: Serves no purpose for this function, only for consistency's - sake - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - facts = get_or_abort(puppetdb.fact_names) - - facts_columns = [[]] - letter = None - letter_list = None - break_size = (len(facts) / 4) + 1 - next_break = break_size - count = 0 - for fact in facts: - count += 1 - - if letter != fact[0].upper() or not letter: - if count > next_break: - # Create a new column - facts_columns.append([]) - next_break += break_size - if letter_list: - facts_columns[-1].append(letter_list) - # Reset - letter = fact[0].upper() - letter_list = [] - - letter_list.append(fact) - facts_columns[-1].append(letter_list) - - return render_template('facts.html', - facts_columns=facts_columns, - envs=envs, - current_env=env) - - -@app.route('/fact/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) -@app.route('//fact/', defaults={'value': None}) -@app.route('/fact//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact//') -def fact(env, fact, value): - """Fetches the specific fact(/value) from PuppetDB and displays per - node for which this fact is known. - - :param env: Searches for facts in this environment - :type env: :obj:`string` - :param fact: Find all facts with this name - :type fact: :obj:`string` - :param value: Find all facts with this value - :type value: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - render_graph = False - if fact in graph_facts and not value: - render_graph = True - - value_safe = value - if value is not None: - value_safe = unquote_plus(value) - - return render_template( - 'fact.html', - fact=fact, - value=value, - value_safe=value_safe, - render_graph=render_graph, - envs=envs, - current_env=env) - - -@app.route('/fact//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node': None, 'value': None}) -@app.route('//fact//json', defaults={'node': None, 'value': None}) -@app.route('/fact///json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) -@app.route('/fact///json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) -@app.route('//fact///json', defaults={'node': None}) -@app.route('/node//facts/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'fact': None, 'value': None}) -@app.route('//node//facts/json', - defaults={'fact': None, 'value': None}) -def fact_ajax(env, node, fact, value): - """Fetches the specific facts matching (node/fact/value) from PuppetDB and - return a JSON table - - :param env: Searches for facts in this environment - :type env: :obj:`string` - :param node: Find all facts for this node - :type node: :obj:`string` - :param fact: Find all facts with this name - :type fact: :obj:`string` - :param value: Filter facts whose value is equal to this - :type value: :obj:`string` - """ - draw = int(request.args.get('draw', 0)) - - envs = environments() - check_env(env, envs) - - render_graph = False - if fact in graph_facts and not value and not node: - render_graph = True - - query = AndOperator() - if node: - query.add(EqualsOperator("certname", node)) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - if len(query.operations) == 0: - query = None - - # Generator needs to be converted (graph / total) - try: - value = int(value) - except ValueError: - if value is not None and query is not None: - if is_bool(value): - query.add(EqualsOperator('value', bool(strtobool(value)))) - else: - query.add(EqualsOperator('value', unquote_plus(value))) - except TypeError: - pass - - facts = [f for f in get_or_abort( - puppetdb.facts, - name=fact, - query=query)] - - total = len(facts) - - counts = {} - json = { - 'draw': draw, - 'recordsTotal': total, - 'recordsFiltered': total, - 'data': []} - - for fact_h in facts: - line = [] - if not fact: - line.append(fact_h.name) - if not node: - line.append('{1}'.format( - url_for('node', env=env, node_name=fact_h.node), - fact_h.node)) - if not value: - fact_value = fact_h.value - if isinstance(fact_value, str): - fact_value = quote_plus(fact_h.value) - - line.append('{1}'.format( - url_for( - 'fact', env=env, fact=fact_h.name, value=fact_value), - fact_h.value)) - - json['data'].append(line) - - if render_graph: - if fact_h.value not in counts: - counts[fact_h.value] = 0 - counts[fact_h.value] += 1 - - if render_graph: - json['chart'] = [ - {"label": "{0}".format(k).replace('\n', ' '), - "value": counts[k]} - for k in sorted(counts, key=lambda k: counts[k], reverse=True)] - - return jsonify(json) \ No newline at end of file diff --git a/puppetboard/forms.py b/puppetboard/forms.py new file mode 100644 index 000000000..e7f969171 --- /dev/null +++ b/puppetboard/forms.py @@ -0,0 +1,36 @@ +from collections import OrderedDict + +from flask_wtf import FlaskForm +from wtforms import (BooleanField, SelectField, TextAreaField, validators) + +from puppetboard.core import get_app + +app = get_app() +QUERY_ENDPOINTS = OrderedDict([ + # PuppetDB API endpoint, Form name + ('pql', 'PQL'), + ('nodes', 'Nodes'), + ('resources', 'Resources'), + ('facts', 'Facts'), + ('factsets', 'Fact Sets'), + ('fact-paths', 'Fact Paths'), + ('fact-contents', 'Fact Contents'), + ('reports', 'Reports'), + ('events', 'Events'), + ('catalogs', 'Catalogs'), + ('edges', 'Edges'), + ('environments', 'Environments'), +]) +ENABLED_QUERY_ENDPOINTS = app.config.get( + 'ENABLED_QUERY_ENDPOINTS', list(QUERY_ENDPOINTS.keys())) + + +class QueryForm(FlaskForm): + """The form used to allow freeform queries to be executed against + PuppetDB.""" + query = TextAreaField('Query', [validators.DataRequired( + message='A query is required.')]) + endpoints = SelectField('API endpoint', choices=[ + (key, value) for key, value in QUERY_ENDPOINTS.items() + if key in ENABLED_QUERY_ENDPOINTS], default='pql') + rawjson = BooleanField('Raw JSON') diff --git a/puppetboard/index.py b/puppetboard/index.py deleted file mode 100644 index 6d02b2911..000000000 --- a/puppetboard/index.py +++ /dev/null @@ -1,170 +0,0 @@ -from datetime import datetime, timedelta - -from flask import render_template -from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, ExtractOperator, FunctionOperator -from pypuppetdb.types import Node -from pypuppetdb.utils import json_to_datetime - -import puppetboard.metrics -import puppetboard.nodes -from puppetboard.app import app, check_env, puppetdb, metric_params, node -from puppetboard.core import environments -from puppetboard.utils import get_db_version, get_or_abort - - -@app.route('/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//') -def index(env): - """This view generates the index page and displays a set of metrics and - latest reports on nodes fetched from PuppetDB. - - :param env: Search for nodes in this (Catalog and Fact) environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - metrics = { - 'num_nodes': 0, - 'num_resources': 0, - 'avg_resources_node': 0, - } - nodes_overview = [] - stats = { - 'changed': 0, - 'unchanged': 0, - 'failed': 0, - 'unreported': 0, - 'noop': 0 - } - - nodes = [] - - node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] - resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] - if env == '*': - query = app.config['OVERVIEW_FILTER'] - - prefix = 'puppetlabs.puppetdb.population' - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - num_nodes = get_or_abort( - puppetboard.metrics.metric, - "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), - version=metric_version) - - if resource_stats_enabled: - num_resources = get_or_abort( - puppetboard.metrics.metric, - "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), - version=metric_version) - - metrics['num_resources'] = num_resources['Value'] - try: - # Compute our own average because avg_resources_node['Value'] - # returns a string of the format "num_resources/num_nodes" - # example: "1234/9" instead of doing the division itself. - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources['Value'] / num_nodes['Value'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - - if not node_status_detail_enabled: - nodes = get_node_status_summary(query) - else: - query = AndOperator() - query.add(EqualsOperator('catalog_environment', env)) - - num_nodes_query = ExtractOperator() - num_nodes_query.add_field(FunctionOperator('count')) - num_nodes_query.add_query(query) - - if app.config['OVERVIEW_FILTER'] is not None: - query.add(app.config['OVERVIEW_FILTER']) - - num_nodes = get_or_abort( - puppetdb._query, - 'nodes', - query=num_nodes_query) - - metrics['num_nodes'] = num_nodes[0]['count'] - - if resource_stats_enabled: - num_resources_query = ExtractOperator() - num_resources_query.add_field(FunctionOperator('count')) - num_resources_query.add_query(EqualsOperator("environment", env)) - - num_resources = get_or_abort( - puppetdb._query, - 'resources', - query=num_resources_query) - - metrics['num_resources'] = num_resources[0]['count'] - try: - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources[0]['count'] / num_nodes[0]['count'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - - if not node_status_detail_enabled: - nodes = get_node_status_summary(query) - - if node_status_detail_enabled: - nodes = get_or_abort(puppetboard.nodes.nodes, - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - else: - stats['unchanged'] += 1 - - if node_status_detail_enabled: - if node.status != 'unchanged': - nodes_overview.append(node) - - return render_template( - 'index.html', - metrics=metrics, - nodes=nodes_overview, - stats=stats, - envs=envs, - current_env=env - ) - - -def get_node_status_summary(inner_query): - node_status_query = ExtractOperator() - node_status_query.add_field('certname') - node_status_query.add_field('report_timestamp') - node_status_query.add_field('latest_report_status') - node_status_query.add_query(inner_query) - - node_status = get_or_abort( - puppetdb._query, - 'nodes', - query=node_status_query - ) - - now = datetime.utcnow() - node_infos = [] - for node_state in node_status: - last_report = json_to_datetime(node_state['report_timestamp']) - last_report = last_report.replace(tzinfo=None) - unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) - certname = node_state['certname'] - - node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, - status_report=node_state['latest_report_status'])) - - return node_infos \ No newline at end of file diff --git a/puppetboard/inventory.py b/puppetboard/inventory.py deleted file mode 100644 index 61f9a7c13..000000000 --- a/puppetboard/inventory.py +++ /dev/null @@ -1,89 +0,0 @@ -from flask import render_template, request -from pypuppetdb.QueryBuilder import AndOperator, OrOperator, EqualsOperator - -import puppetboard.facts -from puppetboard.app import app, check_env -from puppetboard.core import environments - - -def inventory_facts(): - # a list of facts descriptions to go in table header - headers = [] - # a list of inventory fact names - fact_names = [] - - # load the list of items/facts we want in our inventory - try: - inv_facts = app.config['INVENTORY_FACTS'] - except KeyError: - inv_facts = [('Hostname', 'fqdn'), - ('IP Address', 'ipaddress'), - ('OS', 'lsbdistdescription'), - ('Architecture', 'hardwaremodel'), - ('Kernel Version', 'kernelrelease')] - - # generate a list of descriptions and a list of fact names - # from the list of tuples inv_facts. - for desc, name in inv_facts: - headers.append(desc) - fact_names.append(name) - - return headers, fact_names - - -@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//inventory') -def inventory(env): - """Fetch all (active) nodes from PuppetDB and stream a table displaying - those nodes along with a set of facts about them. - - :param env: Search for facts in this environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - headers, fact_names = inventory_facts() - - return render_template( - 'inventory.html', - envs=envs, - current_env=env, - fact_headers=headers) - - -@app.route('/inventory/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//inventory/json') -def inventory_ajax(env): - """Backend endpoint for inventory table""" - draw = int(request.args.get('draw', 0)) - - envs = environments() - check_env(env, envs) - headers, fact_names = inventory_facts() - - query = AndOperator() - fact_query = OrOperator() - fact_query.add([EqualsOperator("name", name) for name in fact_names]) - query.add(fact_query) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - facts = puppetboard.facts.facts(query=query) - - fact_data = {} - for fact in facts: - if fact.node not in fact_data: - fact_data[fact.node] = {} - fact_data[fact.node][fact.name] = fact.value - - total = len(fact_data) - - return render_template( - 'inventory.json.tpl', - draw=draw, - total=total, - total_filtered=total, - fact_data=fact_data, - columns=fact_names) \ No newline at end of file diff --git a/puppetboard/metrics.py b/puppetboard/metrics.py deleted file mode 100644 index 73867c076..000000000 --- a/puppetboard/metrics.py +++ /dev/null @@ -1,89 +0,0 @@ -from urllib.parse import unquote - -from flask import render_template - -from puppetboard.app import app, check_env, puppetdb, metric_params -from puppetboard.core import environments -from puppetboard.utils import get_db_version, get_or_abort - - -@app.route('/metrics', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//metrics') -def metrics(env): - """Lists all available metrics that PuppetDB is aware of. - - :param env: While this parameter serves no function purpose it is required - for the environments template block - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - if metric_version == 'v1': - mbeans = get_or_abort(puppetdb._query, 'mbean') - metrics = list(mbeans.keys()) - elif metric_version == 'v2': - # the list response is a dict in the format: - # { - # "domain1": { - # "property1": { - # ... - # } - # }, - # "domain2": { - # "property2": { - # ... - # } - # } - # } - # The MBean names are the combination of the domain and the properties - # with a ":" in between, example: - # domain1:property1 - # domain2:property2 - # reference: https://jolokia.org/reference/html/protocol.html#list - metrics_domains = get_or_abort(puppetdb.metric) - metrics = [] - # get all of the domains - for domain in list(metrics_domains.keys()): - # iterate over all of the properties in this domain - properties = list(metrics_domains[domain].keys()) - for prop in properties: - # combine the current domain and each property with - # a ":" in between - metrics.append(domain + ':' + prop) - else: - raise ValueError("Unknown metric version {} for database version {}" - .format(metric_version, db_version)) - - return render_template('metrics.html', - metrics=sorted(metrics), - envs=envs, - current_env=env) - - -@app.route('/metric/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//metric/') -def metric(env, metric): - """Lists all information about the metric of the given name. - - :param env: While this parameter serves no function purpose it is required - for the environments template block - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - name = unquote(metric) - metric = get_or_abort(puppetdb.metric, metric, version=metric_version) - return render_template( - 'metric.html', - name=name, - metric=sorted(metric.items()), - envs=envs, - current_env=env) \ No newline at end of file diff --git a/puppetboard/nodes.py b/puppetboard/nodes.py deleted file mode 100644 index a10663275..000000000 --- a/puppetboard/nodes.py +++ /dev/null @@ -1,76 +0,0 @@ -from datetime import datetime, timedelta - -from flask import request, Response, stream_with_context -from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, OrOperator, NullOperator, LessEqualOperator - -from puppetboard.app import app, check_env, puppetdb -from puppetboard.core import environments -from puppetboard.utils import yield_or_stop - - -@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//nodes') -def nodes(env): - """Fetch all (active) nodes from PuppetDB and stream a table displaying - those nodes. - - Downside of the streaming aproach is that since we've already sent our - headers we can't abort the request if we detect an error. Because of this - we'll end up with an empty table instead because of how yield_or_stop - works. Once pagination is in place we can change this but we'll need to - provide a search feature instead. - - :param env: Search for nodes in this (Catalog and Fact) environment - :type env: :obj:`string` - """ - envs = environments() - status_arg = request.args.get('status', '') - check_env(env, envs) - - query = AndOperator() - - if env != '*': - query.add(EqualsOperator("catalog_environment", env)) - - if status_arg in ['failed', 'changed', 'unchanged']: - query.add(EqualsOperator('latest_report_status', status_arg)) - elif status_arg == 'unreported': - unreported = datetime.utcnow() - unreported = (unreported - - timedelta(hours=app.config['UNRESPONSIVE_HOURS'])) - unreported = unreported.replace(microsecond=0).isoformat() - - unrep_query = OrOperator() - unrep_query.add(NullOperator('report_timestamp', True)) - unrep_query.add(LessEqualOperator('report_timestamp', unreported)) - - query.add(unrep_query) - - if len(query.operations) == 0: - query = None - - nodelist = puppetdb.nodes( - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - nodes = [] - for node in yield_or_stop(nodelist): - if status_arg: - if node.status == status_arg: - nodes.append(node) - else: - nodes.append(node) - return Response(stream_with_context( - stream_template('nodes.html', - nodes=nodes, - envs=envs, - current_env=env))) - - -def stream_template(template_name, **context): - app.update_template_context(context) - t = app.jinja_env.get_template(template_name) - rv = t.stream(context) - rv.enable_buffering(5) - return rv \ No newline at end of file diff --git a/puppetboard/query.py b/puppetboard/query.py deleted file mode 100644 index d950d6440..000000000 --- a/puppetboard/query.py +++ /dev/null @@ -1,96 +0,0 @@ -from collections import OrderedDict - -import logging -from flask import session, render_template -from flask_wtf import FlaskForm -from werkzeug.exceptions import abort -from wtforms import (BooleanField, SelectField, TextAreaField, validators) - -from puppetboard.app import app, check_env, puppetdb -from puppetboard.core import environments -from puppetboard.utils import get_or_abort - -numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) -logging.basicConfig(level=numeric_level) -log = logging.getLogger(__name__) - - -QUERY_ENDPOINTS = OrderedDict([ - # PuppetDB API endpoint, Form name - ('pql', 'PQL'), - ('nodes', 'Nodes'), - ('resources', 'Resources'), - ('facts', 'Facts'), - ('factsets', 'Fact Sets'), - ('fact-paths', 'Fact Paths'), - ('fact-contents', 'Fact Contents'), - ('reports', 'Reports'), - ('events', 'Events'), - ('catalogs', 'Catalogs'), - ('edges', 'Edges'), - ('environments', 'Environments'), -]) -ENABLED_QUERY_ENDPOINTS = app.config.get('ENABLED_QUERY_ENDPOINTS', list(QUERY_ENDPOINTS.keys())) - - -class QueryForm(FlaskForm): - """The form used to allow freeform queries to be executed against - PuppetDB.""" - query = TextAreaField('Query', [validators.DataRequired( - message='A query is required.')]) - endpoints = SelectField('API endpoint', choices=[ - (key, value) for key, value in QUERY_ENDPOINTS.items() - if key in ENABLED_QUERY_ENDPOINTS], default='pql') - rawjson = BooleanField('Raw JSON') - - -@app.route('/query', methods=('GET', 'POST'), - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//query', methods=('GET', 'POST')) -def query(env): - """Allows to execute raw, user created querries against PuppetDB. This is - currently highly experimental and explodes in interesting ways since none - of the possible exceptions are being handled just yet. This will return - the JSON of the response or a message telling you what whent wrong / - why nothing was returned. - - :param env: Serves no purpose for the query data but is required for the - select field in the environment block - :type env: :obj:`string` - """ - if not app.config['ENABLE_QUERY']: - log.warn('Access to query interface disabled by administrator.') - abort(403) - - envs = environments() - check_env(env, envs) - - form = QueryForm(meta={ - 'csrf_secret': app.config['SECRET_KEY'], - 'csrf_context': session}) - if form.validate_on_submit(): - if form.endpoints.data not in ENABLED_QUERY_ENDPOINTS: - log.warn('Access to query endpoint %s disabled by administrator.', - form.endpoints.data) - abort(403) - - if form.endpoints.data == 'pql': - query = form.query.data - elif form.query.data[0] == '[': - query = form.query.data - else: - query = '[{0}]'.format(form.query.data) - - result = get_or_abort( - puppetdb._query, - form.endpoints.data, - query=query) - return render_template('query.html', - form=form, - result=result, - envs=envs, - current_env=env) - return render_template('query.html', - form=form, - envs=envs, - current_env=env) \ No newline at end of file diff --git a/puppetboard/radiator.py b/puppetboard/radiator.py deleted file mode 100644 index 1602e8593..000000000 --- a/puppetboard/radiator.py +++ /dev/null @@ -1,106 +0,0 @@ -from flask import request, jsonify, render_template -from pypuppetdb.QueryBuilder import AndOperator, ExtractOperator, EqualsOperator, FunctionOperator - -import puppetboard.metrics -import puppetboard.nodes -from puppetboard.app import app, check_env, puppetdb, metric_params -from puppetboard.core import environments -from puppetboard.utils import get_db_version, get_or_abort - - -@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//radiator') -def radiator(env): - """This view generates a simplified monitoring page - akin to the radiator view in puppet dashboard - """ - envs = environments() - check_env(env, envs) - - if env == '*': - db_version = get_db_version(puppetdb) - query_type, metric_version = metric_params(db_version) - - query = None - metrics = get_or_abort( - puppetboard.metrics.metric, - 'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type, - version=metric_version) - num_nodes = metrics['Value'] - else: - query = AndOperator() - metric_query = ExtractOperator() - - query.add(EqualsOperator("catalog_environment", env)) - metric_query.add_field(FunctionOperator('count')) - metric_query.add_query(query) - - metrics = get_or_abort( - puppetdb._query, - 'nodes', - query=metric_query) - num_nodes = metrics[0]['count'] - - nodes = puppetboard.nodes.nodes( - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True - ) - - stats = { - 'changed_percent': 0, - 'changed': 0, - 'failed_percent': 0, - 'failed': 0, - 'noop_percent': 0, - 'noop': 0, - 'skipped_percent': 0, - 'skipped': 0, - 'unchanged_percent': 0, - 'unchanged': 0, - 'unreported_percent': 0, - 'unreported': 0, - } - - for node in nodes: - if node.status == 'unreported': - stats['unreported'] += 1 - elif node.status == 'changed': - stats['changed'] += 1 - elif node.status == 'failed': - stats['failed'] += 1 - elif node.status == 'noop': - stats['noop'] += 1 - elif node.status == 'skipped': - stats['skipped'] += 1 - else: - stats['unchanged'] += 1 - - try: - stats['changed_percent'] = int(100 * (stats['changed'] / - float(num_nodes))) - stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) - stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) - stats['skipped_percent'] = int(100 * (stats['skipped'] / - float(num_nodes))) - stats['unchanged_percent'] = int(100 * (stats['unchanged'] / - float(num_nodes))) - stats['unreported_percent'] = int(100 * (stats['unreported'] / - float(num_nodes))) - except ZeroDivisionError: - stats['changed_percent'] = 0 - stats['failed_percent'] = 0 - stats['noop_percent'] = 0 - stats['skipped_percent'] = 0 - stats['unchanged_percent'] = 0 - stats['unreported_percent'] = 0 - - if ('Accept' in request.headers and - request.headers["Accept"] == 'application/json'): - return jsonify(**stats) - - return render_template( - 'radiator.html', - stats=stats, - total=num_nodes - ) \ No newline at end of file diff --git a/puppetboard/reports.py b/puppetboard/reports.py deleted file mode 100644 index ef4e59fe0..000000000 --- a/puppetboard/reports.py +++ /dev/null @@ -1,223 +0,0 @@ -import json - -import commonmark -from flask import render_template, request, abort -from itertools import tee -from pypuppetdb.QueryBuilder import AndOperator, EqualsOperator, OrOperator, RegexOperator, GreaterEqualOperator, \ - LessEqualOperator - -import puppetboard.metrics -from puppetboard.app import app, check_env, puppetdb -from puppetboard.core import environments -from puppetboard.utils import get_or_abort, yield_or_stop - - -@app.route('/reports', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node_name': None}) -@app.route('//reports', defaults={'node_name': None}) -@app.route('/reports/', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//reports/') -def reports(env, node_name): - """Query and Return JSON data to reports Jquery datatable - - :param env: Search for all reports in this environment - :type env: :obj:`string` - """ - envs = environments() - check_env(env, envs) - return render_template( - 'reports.html', - envs=envs, - current_env=env, - node_name=node_name, - columns=REPORTS_COLUMNS) - - -@app.route('/reports/json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT'], - 'node_name': None}) -@app.route('//reports/json', defaults={'node_name': None}) -@app.route('/reports//json', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//reports//json') -def reports_ajax(env, node_name): - """Query and Return JSON data to reports Jquery datatable - - :param env: Search for all reports in this environment - :type env: :obj:`string` - """ - draw = int(request.args.get('draw', 0)) - start = int(request.args.get('start', 0)) - length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT'])) - paging_args = {'limit': length, 'offset': start} - search_arg = request.args.get('search[value]') - order_column = int(request.args.get('order[0][column]', 0)) - order_filter = REPORTS_COLUMNS[order_column].get( - 'filter', REPORTS_COLUMNS[order_column]['attr']) - order_dir = request.args.get('order[0][dir]', 'desc') - order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir) - status_args = request.args.get('columns[1][search][value]', '').split('|') - date_args = request.args.get('columns[0][search][value]', '') - max_col = len(REPORTS_COLUMNS) - for i in range(len(REPORTS_COLUMNS)): - if request.args.get("columns[%s][data]" % i, None): - max_col = i + 1 - - envs = environments() - check_env(env, envs) - reports_query = AndOperator() - - if env != '*': - reports_query.add(EqualsOperator("environment", env)) - - if node_name: - reports_query.add(EqualsOperator("certname", node_name)) - - if search_arg: - search_query = OrOperator() - search_query.add(RegexOperator("certname", r"%s" % search_arg)) - search_query.add(RegexOperator("puppet_version", r"%s" % search_arg)) - search_query.add(RegexOperator( - "configuration_version", r"%s" % search_arg)) - reports_query.add(search_query) - - if date_args: - dates = json.loads(date_args) - - if len(dates) > 0: - date_query = AndOperator() - - if 'min' in dates: - date_query.add(GreaterEqualOperator('end_time', dates['min'])) - - if 'max' in dates: - date_query.add(LessEqualOperator('end_time', dates['max'])) - - reports_query.add(date_query) - - status_query = OrOperator() - for status_arg in status_args: - if status_arg in ['failed', 'changed', 'unchanged']: - arg_query = AndOperator() - arg_query.add(EqualsOperator('status', status_arg)) - arg_query.add(EqualsOperator('noop', False)) - status_query.add(arg_query) - if status_arg == 'unchanged': - arg_query = AndOperator() - arg_query.add(EqualsOperator('noop', True)) - arg_query.add(EqualsOperator('noop_pending', False)) - status_query.add(arg_query) - elif status_arg == 'noop': - arg_query = AndOperator() - arg_query.add(EqualsOperator('noop', True)) - arg_query.add(EqualsOperator('noop_pending', True)) - status_query.add(arg_query) - - if len(status_query.operations) == 0: - if len(reports_query.operations) == 0: - reports_query = None - else: - reports_query.add(status_query) - - if status_args[0] != 'none': - reports = get_or_abort( - puppetdb.reports, - query=reports_query, - order_by=order_args, - include_total=True, - **paging_args) - reports, reports_events = tee(reports) - total = None - else: - reports = [] - reports_events = [] - total = 0 - - # Convert metrics to relational dict - metrics = {} - for report in reports_events: - if total is None: - total = puppetdb.total - - metrics[report.hash_] = {} - for m in puppetboard.metrics.metrics: - if m['category'] not in metrics[report.hash_]: - metrics[report.hash_][m['category']] = {} - metrics[report.hash_][m['category']][m['name']] = m['value'] - - if total is None: - total = 0 - - return render_template( - 'reports.json.tpl', - draw=draw, - total=total, - total_filtered=total, - reports=reports, - metrics=metrics, - envs=envs, - current_env=env, - columns=REPORTS_COLUMNS[:max_col]) - - -@app.route('/report//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//report//') -def report(env, node_name, report_id): - """Displays a single report including all the events associated with that - report and their status. - - The report_id may be the puppetdb's report hash or the - configuration_version. This allows for better integration - into puppet-hipchat. - - :param env: Search for reports in this environment - :type env: :obj:`string` - :param node_name: Find the reports whose certname match this value - :type node_name: :obj:`string` - :param report_id: The hash or the configuration_version of the desired - report - :type report_id: :obj:`string` - """ - envs = environments() - check_env(env, envs) - query = AndOperator() - report_id_query = OrOperator() - - report_id_query.add(EqualsOperator("hash", report_id)) - report_id_query.add(EqualsOperator("configuration_version", report_id)) - - if env != '*': - query.add(EqualsOperator("environment", env)) - - query.add(EqualsOperator("certname", node_name)) - query.add(report_id_query) - - reports = puppetdb.reports(query=query) - - try: - report = next(reports) - except StopIteration: - abort(404) - - report.version = commonmark.commonmark(report.version) - - return render_template( - 'report.html', - report=report, - events=yield_or_stop(report.events()), - logs=report.logs, - metrics=puppetboard.metrics.metrics, - envs=envs, - current_env=env) - - -REPORTS_COLUMNS = [ - {'attr': 'end', 'filter': 'end_time', 'name': 'End time', 'type': 'datetime'}, - {'attr': 'status', 'name': 'Status', 'type': 'status'}, - {'attr': 'certname', 'name': 'Certname', 'type': 'node'}, - {'attr': 'version', 'filter': 'configuration_version', 'name': 'Configuration version'}, - {'attr': 'agent_version', 'filter': 'puppet_version', 'name': 'Agent version'}, -] \ No newline at end of file diff --git a/puppetboard/utils.py b/puppetboard/utils.py index 510aead70..b68455117 100644 --- a/puppetboard/utils.py +++ b/puppetboard/utils.py @@ -8,14 +8,10 @@ from pypuppetdb.errors import EmptyResponseError from requests.exceptions import ConnectionError, HTTPError -from puppetboard.app import app -numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None) -logging.basicConfig(level=numeric_level) log = logging.getLogger(__name__) - @contextfunction def url_static_offline(context, value): request_parts = os.path.split(os.path.dirname(context.name))