From 7c027dd97d7c25e095b17c78da47b7c9060d0eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 25 Oct 2013 15:43:14 +0200 Subject: [PATCH 1/4] Add basic support for node status by using the most recent report The following frontend features are implemented - Number of failures, successes, noops/skips in overview - Show latest reports with 1 or more events in overview - Direct links to latest Report - Number and types of events in nodes list --- puppetboard/app.py | 48 ++++++++++++++++++++++---- puppetboard/static/css/puppetboard.css | 13 +++++++ puppetboard/static/js/tables.js | 4 +-- puppetboard/templates/index.html | 46 +++++++++++++++++++++--- puppetboard/templates/nodes.html | 11 ++++++ requirements.txt | 2 +- 6 files changed, 110 insertions(+), 14 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 526dd0b..2808ee5 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -8,7 +8,7 @@ import urllib from flask import ( Flask, render_template, abort, url_for, - Response, stream_with_context, + Response, stream_with_context, redirect, ) from pypuppetdb import connect @@ -71,8 +71,9 @@ def server_error(e): @app.route('/') def index(): - """This view generates the index page and displays a set of metrics fetched - from PuppetDB.""" + """This view generates the index page and displays a set of metrics and latest reports on + nodes fetched from PuppetDB. + """ # TODO: Would be great if we could parallelize this somehow, doing these # requests in sequence is rather pointless. num_nodes = get_or_abort(puppetdb.metric, @@ -85,6 +86,8 @@ def index(): 'com.puppetlabs.puppetdb.command:type=global,name=fatal') mean_command_time = get_or_abort(puppetdb.metric, 'com.puppetlabs.puppetdb.command:type=global,name=processing-time') + mean_command_time = get_or_abort(puppetdb.metric, + 'com.puppetlabs.puppetdb.command:type=global,name=processing-time') metrics = { 'num_nodes': num_nodes['Value'], 'num_resources': num_resources['Value'], @@ -92,7 +95,13 @@ def index(): 'mean_failed_commands': mean_failed_commands['MeanRate'], 'mean_command_time': "{0:10.6f}".format(mean_command_time['MeanRate']), } - return render_template('index.html', metrics=metrics) + + latest_event_count = puppetdb._query('aggregate-event-counts', query='["=", "latest-report?", true]', summarize_by='certname') + latest_event_count['noopskip'] = latest_event_count['noops']+latest_event_count['skips'] + + latest_events = puppetdb._query('event-counts', query='["=", "latest-report?", true]', summarize_by='certname') + + return render_template('index.html', metrics=metrics, latest_event_count=latest_event_count, latest_events=latest_events) @app.route('/nodes') def nodes(): @@ -105,8 +114,19 @@ def nodes(): works. Once pagination is in place we can change this but we'll need to provide a search feature instead. """ + latest_events = puppetdb._query('event-counts', query='["=", "latest-report?", true]', summarize_by='certname') + nodes = [] + for node in yield_or_stop(puppetdb.nodes()): + # check if node name is contained in any of the event-counts (grouped by certname) + status = [s for s in latest_events if s['subject']['title'] == node.name] + if status: + node.status = status[0] + else: + node.status = None + nodes.append(node) + return Response(stream_with_context(stream_template('nodes.html', - nodes=yield_or_stop(puppetdb.nodes())))) + nodes=nodes))) @app.route('/node/') def node(node_name): @@ -146,10 +166,26 @@ def reports_node(node): return render_template('reports_node.html', reports=reports, nodename=node) +@app.route('/report/latest/') +def report_latest(node_name): + """Redirect to the latest report of a given node. This is a workaround + as long as PuppetDB can't filter reports for latest-report? field. This + feature has been requested: http://projects.puppetlabs.com/issues/21554 + """ + # TODO: use limit parameter in _query to get just one report + node = get_or_abort(puppetdb.node, node_name) + if app.config['PUPPETDB_API'] > 2: + reports = ten_reports(node.reports()) + else: + reports = iter([]) + report = list(yield_or_stop(reports))[0] + return redirect(url_for('report', node=node_name, report_id=report)) + @app.route('/report//') def report(node, report_id): """Displays a single report including all the events associated with that - report and their status.""" + report and their status. + """ if app.config['PUPPETDB_API'] > 2: reports = puppetdb.reports('["=", "certname", "{0}"]'.format(node)) else: diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index 7cd5098..f535a8c 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -52,3 +52,16 @@ th.headerSortDown:after { .table tbody tr.error>td { background-color: #f2dede; } +h1.error { + color: rgb(223, 46, 27); +} +h1.success { + color: #18BC9C; +} +h1.noop { + color:#aaa; +} +.numtag { + width:20px; + text-align:center; +} diff --git a/puppetboard/static/js/tables.js b/puppetboard/static/js/tables.js index 4628d49..7fb44b8 100644 --- a/puppetboard/static/js/tables.js +++ b/puppetboard/static/js/tables.js @@ -8,11 +8,11 @@ $('.nodes').tablesorter({ headers: { - 3: { + 4: { sorter: false } }, - sortList: [[0, 0]] + sortList: [[1, 0]] }); $('.facts').tablesorter({ diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index 4e41277..acf561c 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -1,11 +1,27 @@ {% extends 'layout.html' %} {% block row_fluid %} -
-
- We need something fancy here. -
-
+
+

Statistics

+
+
+
+
+
+

{{latest_event_count['failures']}} Failures

+ in the latest node reports +
+
+

{{latest_event_count['successes']}} Successes

+ in the latest node reports +
+
+

{{latest_event_count['noopskip']}} Noop/Skip

+ in the latest node reports +
+
+
+
@@ -34,5 +50,25 @@
+ +
+
+

Latest Reports with events

+
+ + + {% for event in latest_events %} + + + + + + + + {% endfor %} + +
{% if event['failures'] %}{{event['failures']}}{% endif%}{% if event['successes'] %}{{event['successes']}}{% endif%}{{event['subject']['title']}}Latest Report
+
+
{% endblock row_fluid %} diff --git a/puppetboard/templates/nodes.html b/puppetboard/templates/nodes.html index 9d483a7..bb879c8 100644 --- a/puppetboard/templates/nodes.html +++ b/puppetboard/templates/nodes.html @@ -18,6 +18,7 @@ + {% if config.PUPPETDB_API > 2 %} @@ -29,6 +30,16 @@ {% for node in nodes %} + {% if config.PUPPETDB_API > 2 %} diff --git a/requirements.txt b/requirements.txt index cab4027..f3975d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ MarkupSafe==0.18 WTForms==1.0.4 Werkzeug==0.9.3 itsdangerous==0.22 -pypuppetdb==0.0.4 +git+git://github.com/juliushaertl/pypuppetdb.git@feature-v3-newapis#egg=pypuppetdb requests==1.2.3 From c7bae2efa3fa4680ba636fd18cc5060a6350fb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 25 Oct 2013 16:07:00 +0200 Subject: [PATCH 2/4] fix for required branch of pypuppetdb with new api support --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f3975d1..ae82b13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ MarkupSafe==0.18 WTForms==1.0.4 Werkzeug==0.9.3 itsdangerous==0.22 -git+git://github.com/juliushaertl/pypuppetdb.git@feature-v3-newapis#egg=pypuppetdb +git+https://github.com/juliushaertl/pypuppetdb.git@feature-v3-newapis#egg=pypuppetdb requests==1.2.3 From 5ca758dd397dfc61b0db1a94aa9831adb9c61c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 28 Oct 2013 11:36:37 +0100 Subject: [PATCH 3/4] show list of nodes without report for x hours in overview - the amount of hours is defined as `UNRESPONSIVE_HOURS` in default_settings.py - small status layout improvement in nodes list - latest report button in nodes list - nedap/pypuppetdb repo as requirement ( new api was merged nedap/pypuppetdb#17 ) --- puppetboard/app.py | 11 +++++- puppetboard/default_settings.py | 1 + puppetboard/static/css/puppetboard.css | 5 +++ puppetboard/templates/index.html | 53 ++++++++++++++++---------- puppetboard/templates/nodes.html | 17 ++++----- requirements.txt | 2 +- 6 files changed, 57 insertions(+), 32 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 2808ee5..a871b68 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -5,6 +5,7 @@ import os import logging import collections import urllib +from datetime import datetime, timedelta from flask import ( Flask, render_template, abort, url_for, @@ -101,7 +102,15 @@ def index(): latest_events = puppetdb._query('event-counts', query='["=", "latest-report?", true]', summarize_by='certname') - return render_template('index.html', metrics=metrics, latest_event_count=latest_event_count, latest_events=latest_events) + unreported = [] + for node in puppetdb.nodes(): + node_last_seen = node.report_timestamp.replace(tzinfo=None) + if node_last_seen < (datetime.utcnow()-timedelta(hours=app.config['UNRESPONSIVE_HOURS'])): + delta = (datetime.utcnow()-node_last_seen) + node.noresponse = str(delta.days) + "d " + str(int(delta.seconds/3600)) +"h " + str(int((delta.seconds%3600)/60))+ "m" + unreported.append(node) + + return render_template('index.html', metrics=metrics, latest_event_count=latest_event_count, latest_events=latest_events, unreported=unreported, unreported_time=app.config['UNRESPONSIVE_HOURS']) @app.route('/nodes') def nodes(): diff --git a/puppetboard/default_settings.py b/puppetboard/default_settings.py index d9dd533..0421bdd 100644 --- a/puppetboard/default_settings.py +++ b/puppetboard/default_settings.py @@ -7,5 +7,6 @@ PUPPETDB_TIMEOUT=20 PUPPETDB_API=3 DEV_LISTEN_HOST='127.0.0.1' DEV_LISTEN_PORT=5000 +UNRESPONSIVE_HOURS=2 ENABLE_QUERY=True LOGLEVEL='info' diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index f535a8c..49d18d1 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -65,3 +65,8 @@ h1.noop { width:20px; text-align:center; } +.label-nothing { + background-color:#ddd; + width:20px; + color:#ddd; +} diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index acf561c..e11489b 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -8,16 +8,16 @@
-

{{latest_event_count['failures']}} Failures

- in the latest node reports +

{{latest_event_count['failures']}} Failed

+ nodes in the latest reports
-

{{latest_event_count['successes']}} Successes

- in the latest node reports +

{{latest_event_count['successes']}} Succeeded

+ nodes in the latest reports
-

{{latest_event_count['noopskip']}} Noop/Skip

- in the latest node reports +

{{ unreported|length }} Unreported

+ nodes in the last {{ unreported_time }} hours
@@ -53,21 +53,34 @@
+

Nodes without Report since {{ unreported_time }} hours

+
Status Hostname Catalog compiled at
+ {% if node.status %} + {% if node.status['failures'] %}{{node.status['failures']}}{% endif %} + {% if node.status['successes'] %}{{node.status['successes']}}{% endif %} + {% if node.status['noops'] or node.status['skips'] %}{{node.status['skips']+node.status['noops']}}{% endif %} + {% else %} + 0 + {% endif %} + + {{node.name}} {{node.catalog_timestamp}}
+ + {% for node in unreported %} + + + + + + + {% endfor %} + +
No Report{{ node.noresponse }}{{ node.name }}Latest Report

Latest Reports with events

-
- - - {% for event in latest_events %} - - - - - - - - {% endfor %} - -
{% if event['failures'] %}{{event['failures']}}{% endif%}{% if event['successes'] %}{{event['successes']}}{% endif%}{{event['subject']['title']}}Latest Report
+ + + {% for event in latest_events %} + + + + + + {% endfor %} + +
+ {% if event['failures'] %}{{event['failures']}}{% else %}0{% endif%} + {% if event['successes'] %}{{event['successes']}}{% else %}0{% endif%} + {{event['subject']['title']}}Latest Report
diff --git a/puppetboard/templates/nodes.html b/puppetboard/templates/nodes.html index bb879c8..c1e30d5 100644 --- a/puppetboard/templates/nodes.html +++ b/puppetboard/templates/nodes.html @@ -18,7 +18,7 @@ - + {% if config.PUPPETDB_API > 2 %} @@ -30,15 +30,11 @@ {% for node in nodes %} - @@ -52,6 +48,7 @@ diff --git a/requirements.txt b/requirements.txt index ae82b13..1b3318c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ MarkupSafe==0.18 WTForms==1.0.4 Werkzeug==0.9.3 itsdangerous==0.22 -git+https://github.com/juliushaertl/pypuppetdb.git@feature-v3-newapis#egg=pypuppetdb +git+https://github.com/nedap/pypuppetdb#egg=pypuppetdb requests==1.2.3 From bbb65939c9be7e61e9f0b5115de757f17b793bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Mon, 28 Oct 2013 15:38:45 +0100 Subject: [PATCH 4/4] remove duplicate mean_command_time --- puppetboard/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index a871b68..66e3bd8 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -87,8 +87,6 @@ def index(): 'com.puppetlabs.puppetdb.command:type=global,name=fatal') mean_command_time = get_or_abort(puppetdb.metric, 'com.puppetlabs.puppetdb.command:type=global,name=processing-time') - mean_command_time = get_or_abort(puppetdb.metric, - 'com.puppetlabs.puppetdb.command:type=global,name=processing-time') metrics = { 'num_nodes': num_nodes['Value'], 'num_resources': num_resources['Value'],
StatusStatus Hostname Catalog compiled at
- {% if node.status %} - {% if node.status['failures'] %}{{node.status['failures']}}{% endif %} - {% if node.status['successes'] %}{{node.status['successes']}}{% endif %} - {% if node.status['noops'] or node.status['skips'] %}{{node.status['skips']+node.status['noops']}}{% endif %} - {% else %} - 0 - {% endif %} - + + {% if node.status['failures'] %}{{node.status['failures']}}{% else %}0{% endif %} + {% if node.status['successes'] %}{{node.status['successes']}}{% else %}0{% endif %} + {% if node.status['noops'] or node.status['skips'] %}{{node.status['skips']+node.status['noops']}}{% else %}0{% endif %} + {{node.name}} {{node.catalog_timestamp}} {% if node.report_timestamp %} + Latest Report Reports {% endif %}