diff --git a/README.md b/README.md index f0322d14a..8f898bfd7 100644 --- a/README.md +++ b/README.md @@ -123,51 +123,73 @@ For information about how to generate the correct keys please refer to the [pypuppetdb documentation](https://pypuppetdb.readthedocs.io/en/latest/connecting.html#ssl). Alternatively it is possible to explicitly specify the protocol to be used setting the `PUPPETDB_PROTO` variable. -Other settings that might be interesting in no particular order: +Other settings that might be interesting: -- `SECRET_KEY`: Refer to [Flask documentation](https://flask.palletsprojects.com/en/1.1.x/quickstart/#sessions), - section "How to generate good secret keys" for more info. Defaults to a random 24-char string generated by - `os.random(24)`. -- `PUPPETDB_TIMEOUT`: Defaults to 20 seconds but you might need to increase this value. It depends on how big the - results are when querying PuppetDB. This behaviour will change in a future release when pagination will be introduced. -- `UNRESPONSIVE_HOURS`: The amount of hours since the last check-in after which a node is considered unresponsive. -- `LOGLEVEL`: A string representing the loglevel. It defaults to `'info'` but can be changed to `'warning'` or - `'critical'` for less verbose logging or `'debug'` for more information. -- `ENABLE_QUERY`: Defaults to `True` causing a Query tab to show up in the web interface allowing users to write - and execute arbitrary queries against a set of endpoints in PuppetDB. Change this to `False` to disable this. - See `ENABLED_QUERY_ENDPOINTS` to fine-tune which endpoints are allowed. -- `ENABLED_QUERY_ENDPOINTS`: If `ENABLE_QUERY` is `True`, allow to fine tune the endpoints of PuppetDB APIs that - can be queried. It must be a list of strings of PuppetDB endpoints for which the query is enabled. - See the `QUERY_ENDPOINTS` constant in the `puppetboard.app` module for a list of the available endpoints. -- `GRAPH_TYPE`: Specify the type of graph to display. Default is - pie, other good option is donut. Other choices can be found here: - \_C3JS\_documentation\` -- `GRAPH_FACTS`: A list of fact names to tell PuppetBoard to generate a pie-chart on the fact page. With some fact - values being unique per node, like ipaddress, uuid, and serial number, as well as structured facts it was no longer - feasible to generate a graph for everything. -- `INVENTORY_FACTS`: A list of tuples that serve as the column header and the fact name to search for to create +#### General UI + +* `LOCALISE_TIMESTAMP`: Make the timestamps use the timezone provided by your browser. Defaults to `True`. +* `REFRESH_RATE`: The number of seconds between automatic page refreshes. Defaults to `30`. Set to `0` to disable. +* `DEFAULT_ENVIRONMENT`: The default value of the Puppet environment filter. Defaults to `'production'`. Set to `'*'` + to show all environments. +* `INVENTORY_FACTS`: A list of tuples that serve as the column header and the fact name to search for to create the inventory page. If a fact is not found for a node then `undef` is printed. -- `ENABLE_CATALOG`: If set to `True` allows the user to view a node's latest catalog. This includes all managed +* `ENABLE_QUERY`: If enabled then the Query tab is available which allows users to send arbitrary queries against + a set of endpoints in PuppetDB. Defaults to `True`. +* `ENABLED_QUERY_ENDPOINTS`: If `ENABLE_QUERY` is `True`, it allows restricting the list of the available endpoints. + It must be a list of strings of PuppetDB endpoints for which the query is enabled. The default is empty, which + means that all endpoints are allowed. See the `QUERY_ENDPOINTS` constant in `forms.py` for a list of the available + endpoints. +* `ENABLE_CATALOG`: If enabled then it allows the user to view a node's latest catalog. This includes all managed resources, their file-system locations and their relationships, if available. Defaults to `False`. -- `REFRESH_RATE`: Defaults to `30` the number of seconds to wait until the index page is automatically refreshed. -- `DEFAULT_ENVIRONMENT`: Defaults to `'production'`, as the name suggests, load all information filtered by this - environment value. -- `REPORTS_COUNT`: Defaults to `10` the limit of the number of reports to load on the node or any reports page. -- `OFFLINE_MODE`: If set to `True` load static assets (jquery, semantic-ui, etc) from the local web server instead - of a CDN. Defaults to `False`. -- `DAILY_REPORTS_CHART_ENABLED`: Enable the use of daily chart graphs when looking at dashboard and node view. -- `DAILY_REPORTS_CHART_DAYS`: Number of days to show history for on the daily report graphs. -- `DISPLAYED_METRICS`: Metrics to show when displaying node summary. Example: `'resources.total'`, `'events.noop'`. -- `TABLE_COUNT_SELECTOR`: Configure the dropdown to limit number of hosts to show per page. -- `LITTLE_TABLE_COUNT`: Default number of reports to show when when looking at a node. -- `NORMAL_TABLE_COUNT`: Default number of nodes to show when displaying reports and catalog nodes. -- `LOCALISE_TIMESTAMP`: Normalize time based on localserver time. -- `WITH_EVENT_NUMBERS`: If set to `True` then Overview and Nodes list shows exact number of changed resources + +#### Overview page + +* `RESOURCES_STATS_ENABLED`: If set then the section with "Resources managed" and "Avg. resources/node" + of the Overview page is shown. Defaults to `True`. +* `DAILY_REPORTS_CHART_ENABLED`: Enable the use of daily chart graphs when looking at Overview and Node view. + Defaults to `True`. +* `DAILY_REPORTS_CHART_DAYS`: Number of days to show history for on the daily report graphs. Defaults to `8`. +* `NODES_STATUS_DETAIL_ENABLED`: If set then such section of the Overview page is shown. Defaults to `True`. +* `UNRESPONSIVE_HOURS`: The amount of hours since the last check-in after which a node is considered unreported. + Defaults to `2`. + +#### Lists of nodes (Overview and Nodes pages) + +* `WITH_EVENT_NUMBERS`: If set to `True` then Overview and Nodes list shows exact number of changed resources in the last report. Otherwise shows only 'some' string if there are resources with given status. Setting this to `False` gives performance benefits, especially in big Puppet environments (more than few hundreds of nodes). Defaults to `True`. -- `DEV_LISTEN_HOST`: For use with dev.py for development. Default is localhost -- `DEV_LISTEN_PORT`: For use with dev.py for development. Default is 5000 +* `DISPLAYED_METRICS`: Metrics to show as the node "Status", to the right of the last report result. + Defaults to `['resources.total', 'events.failure', 'events.success', 'resources.skipped', 'events.noop']`. + +#### Facts page + +* `GRAPH_FACTS`: A list of fact names to tell Puppetboard to generate a pie-chart on the fact page. With some fact + values being unique per node, like ipaddress, uuid, and serial number, as well as structured facts it was no longer + feasible to generate a graph for everything. +* `GRAPH_TYPE`: Specify the type of graph to display. Default is pie, other good option is donut. Other choices + can be found here: [C3.js examples](https://c3js.org/examples.html) + +#### Various UI + +* `LITTLE_TABLE_COUNT`: Default number of reports to show when looking at a node. Defaults to `10`. +* `NORMAL_TABLE_COUNT`: Default number of nodes to show when displaying reports and catalog nodes. Defaults to `100`. +* `TABLE_COUNT_SELECTOR`: Configure the dropdown to limit number of hosts to show per page. + Defaults to `[10, 20, 50, 100, 500]`. + +#### Other + +* `PUPPETDB_TIMEOUT`: Defaults to 20 seconds but you might need to increase this value. It depends on how big the + results are when querying PuppetDB. This behaviour will change in a future release when pagination will be introduced. +* `LOGLEVEL`: A string representing the loglevel. It defaults to `'info'` but can be changed to `'warning'` or + `'critical'` for less verbose logging or `'debug'` for more information. +* `SECRET_KEY`: Refer to [Flask documentation](https://flask.palletsprojects.com/en/1.1.x/quickstart/#sessions), + section "How to generate good secret keys" for more info. Defaults to a random 24-char string generated by + `os.random(24)`. +* `OFFLINE_MODE`: If set to `True` load static assets (jquery, semantic-ui, etc) from the local web server instead + of a CDN. Defaults to `False`. +* `DEV_LISTEN_HOST`: For use with dev.py for development. Default is localhost +* `DEV_LISTEN_PORT`: For use with dev.py for development. Default is 5000 ## Getting Help diff --git a/docs/EL7.md b/docs/EL7.md index e09ece5a3..669116c04 100644 --- a/docs/EL7.md +++ b/docs/EL7.md @@ -150,7 +150,6 @@ class profile::puppetboard { puppetdb_key => "${ssl_dir}/private_keys/${puppetboard_certname}.pem", puppetdb_ssl_verify => "${ssl_dir}/certs/ca.pem", puppetdb_cert => "${ssl_dir}/certs/${puppetboard_certname}.pem", - reports_count => 40, } class { '::apache::mod::wsgi': diff --git a/puppetboard/app.py b/puppetboard/app.py index f243d2aca..1b9a6e8f5 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -19,6 +19,9 @@ LessEqualOperator, RegexOperator, GreaterEqualOperator) +from pypuppetdb.types import Node +from pypuppetdb.utils import json_to_datetime + from puppetboard.forms import ENABLED_QUERY_ENDPOINTS, QueryForm from puppetboard.utils import (get_or_abort, yield_or_stop, get_db_version, is_bool) @@ -136,12 +139,26 @@ def index(env): :type env: :obj:`string` """ envs = environments() + check_env(env, envs) + metrics = { 'num_nodes': 0, 'num_resources': 0, - 'avg_resources_node': 0} - check_env(env, envs) + 'avg_resources_node': 0, + } + nodes_overview = [] + stats = { + 'changed': 0, + 'unchanged': 0, + 'failed': 0, + 'unreported': 0, + 'noop': 0 + } + + nodes = [] + node_status_detail_enabled = app.config['NODES_STATUS_DETAIL_ENABLED'] + resource_stats_enabled = app.config['RESOURCES_STATS_ENABLED'] if env == '*': query = app.config['OVERVIEW_FILTER'] @@ -153,21 +170,25 @@ def index(env): puppetdb.metric, "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type), version=metric_version) - num_resources = get_or_abort( - puppetdb.metric, - "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), - version=metric_version) - metrics['num_nodes'] = num_nodes['Value'] - metrics['num_resources'] = num_resources['Value'] - try: - # Compute our own average because avg_resources_node['Value'] - # returns a string of the format "num_resources/num_nodes" - # example: "1234/9" instead of doing the division itself. - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources['Value'] / num_nodes['Value'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 + if resource_stats_enabled: + num_resources = get_or_abort( + puppetdb.metric, + "{0}{1}".format(prefix, ':%sname=num-resources' % query_type), + version=metric_version) + + metrics['num_resources'] = num_resources['Value'] + try: + # Compute our own average because avg_resources_node['Value'] + # returns a string of the format "num_resources/num_nodes" + # example: "1234/9" instead of doing the division itself. + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources['Value'] / num_nodes['Value'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) else: query = AndOperator() query.add(EqualsOperator('catalog_environment', env)) @@ -179,40 +200,39 @@ def index(env): if app.config['OVERVIEW_FILTER'] is not None: query.add(app.config['OVERVIEW_FILTER']) - num_resources_query = ExtractOperator() - num_resources_query.add_field(FunctionOperator('count')) - num_resources_query.add_query(EqualsOperator("environment", env)) - num_nodes = get_or_abort( puppetdb._query, 'nodes', query=num_nodes_query) - num_resources = get_or_abort( - puppetdb._query, - 'resources', - query=num_resources_query) + metrics['num_nodes'] = num_nodes[0]['count'] - metrics['num_resources'] = num_resources[0]['count'] - try: - metrics['avg_resources_node'] = "{0:10.0f}".format( - (num_resources[0]['count'] / num_nodes[0]['count'])) - except ZeroDivisionError: - metrics['avg_resources_node'] = 0 - nodes = get_or_abort(puppetdb.nodes, - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True, - with_event_numbers=app.config['WITH_EVENT_NUMBERS']) - - nodes_overview = [] - stats = { - 'changed': 0, - 'unchanged': 0, - 'failed': 0, - 'unreported': 0, - 'noop': 0 - } + if resource_stats_enabled: + num_resources_query = ExtractOperator() + num_resources_query.add_field(FunctionOperator('count')) + num_resources_query.add_query(EqualsOperator("environment", env)) + + num_resources = get_or_abort( + puppetdb._query, + 'resources', + query=num_resources_query) + + metrics['num_resources'] = num_resources[0]['count'] + try: + metrics['avg_resources_node'] = "{0:10.0f}".format( + (num_resources[0]['count'] / num_nodes[0]['count'])) + except ZeroDivisionError: + metrics['avg_resources_node'] = 0 + + if not node_status_detail_enabled: + nodes = get_node_status_summary(query) + + if node_status_detail_enabled: + nodes = get_or_abort(puppetdb.nodes, + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True, + with_event_numbers=app.config['WITH_EVENT_NUMBERS']) for node in nodes: if node.status == 'unreported': @@ -226,8 +246,9 @@ def index(env): else: stats['unchanged'] += 1 - if node.status != 'unchanged': - nodes_overview.append(node) + if node_status_detail_enabled: + if node.status != 'unchanged': + nodes_overview.append(node) return render_template( 'index.html', @@ -239,6 +260,33 @@ def index(env): ) +def get_node_status_summary(inner_query): + node_status_query = ExtractOperator() + node_status_query.add_field('certname') + node_status_query.add_field('report_timestamp') + node_status_query.add_field('latest_report_status') + node_status_query.add_query(inner_query) + + node_status = get_or_abort( + puppetdb._query, + 'nodes', + query=node_status_query + ) + + now = datetime.utcnow() + node_infos = [] + for node_state in node_status: + last_report = json_to_datetime(node_state['report_timestamp']) + last_report = last_report.replace(tzinfo=None) + unreported_border = now - timedelta(hours=app.config['UNRESPONSIVE_HOURS']) + certname = node_state['certname'] + + node_infos.append(Node(node, name=certname, unreported=last_report < unreported_border, + status_report=node_state['latest_report_status'])) + + return node_infos + + @app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//nodes') def nodes(env): @@ -628,48 +676,28 @@ def facts(env): check_env(env, envs) facts = get_or_abort(puppetdb.fact_names) - # we consider a column label to count for ~5 lines - column_label_height = 5 - - # 1 label per different letter and up to 3 more labels for letters spanning - # multiple columns. - column_label_count = 3 + len(set(map(lambda fact: fact[0].upper(), facts))) - - break_size = (len(facts) + column_label_count * column_label_height) / 4.0 - next_break = break_size - - facts_columns = [] - facts_current_column = [] - facts_current_letter = [] + facts_columns = [[]] letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size count = 0 - for fact in facts: count += 1 - if count > next_break: - next_break += break_size - if facts_current_letter: - facts_current_column.append(facts_current_letter) - if facts_current_column: - facts_columns.append(facts_current_column) - facts_current_column = [] - facts_current_letter = [] - letter = None - - if letter != fact[0].upper(): - if facts_current_letter: - facts_current_column.append(facts_current_letter) - facts_current_letter = [] + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset letter = fact[0].upper() - count += column_label_height - - facts_current_letter.append(fact) + letter_list = [] - if facts_current_letter: - facts_current_column.append(facts_current_letter) - if facts_current_column: - facts_columns.append(facts_current_column) + letter_list.append(fact) + facts_columns[-1].append(letter_list) return render_template('facts.html', facts_columns=facts_columns, diff --git a/puppetboard/default_settings.py b/puppetboard/default_settings.py index a9d48b1f8..e8c533d9a 100644 --- a/puppetboard/default_settings.py +++ b/puppetboard/default_settings.py @@ -53,3 +53,5 @@ DAILY_REPORTS_CHART_ENABLED = True DAILY_REPORTS_CHART_DAYS = 8 WITH_EVENT_NUMBERS = True +RESOURCES_STATS_ENABLED = True +NODES_STATUS_DETAIL_ENABLED = True diff --git a/puppetboard/docker_settings.py b/puppetboard/docker_settings.py index 6175d6322..5a210bc16 100644 --- a/puppetboard/docker_settings.py +++ b/puppetboard/docker_settings.py @@ -39,7 +39,7 @@ def coerce_bool(v, default): LOCALISE_TIMESTAMP = coerce_bool(os.getenv('LOCALISE_TIMESTAMP'), True) LOGLEVEL = os.getenv('LOGLEVEL', 'info') -NORMAL_TABLE_COUNT = int(os.getenv('REPORTS_COUNT', '100')) +NORMAL_TABLE_COUNT = int(os.getenv('NORMAL_TABLE_COUNT', '100')) LITTLE_TABLE_COUNT = int(os.getenv('LITTLE_TABLE_COUNT', '10')) TABLE_COUNT_DEF = "10,20,50,100,500" @@ -95,3 +95,7 @@ def coerce_bool(v, default): DAILY_REPORTS_CHART_DAYS = int(os.getenv('DAILY_REPORTS_CHART_DAYS', '8')) WITH_EVENT_NUMBERS = coerce_bool(os.getenv('WITH_EVENT_NUMBERS'), True) + +RESOURCES_STATS_ENABLED = coerce_bool(os.getenv('RESOURCES_STATS_ENABLED'), True) + +NODES_STATUS_DETAIL_ENABLED = coerce_bool(os.getenv('NODES_STATUS_DETAIL_ENABLED'), True) diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index 711980583..17012c9cf 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -57,6 +57,10 @@ h1.ui.header.no-margin-bottom { background-color: #4572A7; } +.ui.header.population { + color: #2C3E50; +} + .ui.header.unreported { color: #3D96AE; } diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index f22309958..586417359 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -13,7 +13,7 @@ {% endif %}
-
+

@@ -50,13 +50,24 @@

unreported in the last {{ config.UNRESPONSIVE_HOURS }} hours

+
+

+ {{ metrics['num_nodes'] }} + {% if metrics['num_nodes'] == 1 %} node {% else %} nodes {% endif %} +

+ total population +
-
+ {% if config.RESOURCES_STATS_ENABLED %} +
+
-

{{metrics['num_nodes']}}

- Population + +
+
+

{{metrics['num_resources']}}

@@ -67,11 +78,13 @@

{{metrics['avg_resources_node']} Avg. resources/node

+ {% endif %} {% if config.DAILY_REPORTS_CHART_ENABLED %}
{% endif %} + {% if config.NODES_STATUS_DETAIL_ENABLED %}
@@ -125,5 +138,6 @@

Nodes status detail

{% endif %}
+ {% endif %}
{% endblock content %} diff --git a/test/test_app.py b/test/test_app.py index f9669b9b2..f21fc2fc3 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -169,10 +169,9 @@ def test_index_all(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '63' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '63' + assert vals[1].string == ' 6' assert rv.status_code == 200 @@ -219,10 +218,9 @@ def test_index_all_puppetdb_v4(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '63' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '63' + assert vals[1].string == ' 6' assert rv.status_code == 200 @@ -269,10 +267,9 @@ def test_index_all_puppetdb_v3(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[0].string == '10' - assert vals[1].string == '60' - assert vals[2].string == ' 6' + assert len(vals) == 2 + assert vals[0].string == '60' + assert vals[1].string == ' 6' assert rv.status_code == 200 @@ -298,8 +295,8 @@ def test_index_division_by_zero(client, mocker, vals = soup.find_all('h1', {"class": "ui header darkblue no-margin-bottom"}) - assert len(vals) == 3 - assert vals[2].string == '0' + assert len(vals) == 2 + assert vals[1].string == '0' def test_offline_mode(client, mocker, @@ -769,7 +766,7 @@ def test_facts_view_empty_when_no_facts(client, searchable = soup.find('div', {'class': 'searchable'}) vals = searchable.find_all('div', {'class': 'column'}) - assert len(vals) == 0 + assert len(vals) == 1 def test_fact_view_with_graph(client, mocker,