Overview, Node pages: Add bar chart of daily runs

The Overview will display a bar chart of daily runs, categorized by
report status (changed, unchanged, failed).

The chart data is loaded asynchronously from JSON so it doesn't provoke
a delay in the page load. The data is JSON enconded.

This feature was in the original Puppet Dashboard.  The change was
proposed and discussed in issue #308 .

Application changes:

- app.py: New view daily_reports_chart to serve the chart data as JSON.

- dailychart.py: Submodule to query and format the chart data.

Template changes:

- layout.html: New block to add more elements to the HTML header.

- index.html, node.html: Add C3 CSS in header block, add DIV placeholder
  for the chart in content block, add dailychart.js (and dependencies)
  in script block.

Settings:

- DAILY_REPORTS_CHART_ENABLED: New setting to turn off the charts. By
  default is on.

- DAILY_REPORTS_CHART_DAYS: Changes the range of days to display in the
  charts.

Javascript changes:

- dailychart.js: New script that loads the JSON data for the chart and
  calls C3 to generate a bar chart.

CSS changes:

- puppetboard.css: Set fixed height to the chart container to avoid a
  page resize after the chart is loaded.

puppetboard/app.py

@@ -13,7 +13,7 @@ from itertools import tee
 from flask import (
     Flask, render_template, abort, url_for,
     Response, stream_with_context, redirect,
-    request, session
+    request, session, jsonify
 )
 
 from pypuppetdb import connect
@@ -24,6 +24,7 @@ from puppetboard.utils import (
     get_or_abort, yield_or_stop,
     jsonprint, prettyprint, Pagination
 )
+from puppetboard.dailychart import get_daily_reports_chart
 
 import werkzeug.exceptions as ex
 
@@ -1109,3 +1110,23 @@ def radiator(env):
         stats=stats,
         total=num_nodes
     )
+
+
+@app.route('/daily_reports_chart.json',
+           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
+@app.route('/<env>/daily_reports_chart.json')
+def daily_reports_chart(env):
+    """Return JSON data to generate a bar chart of daily runs.
+
+    If certname is passed as GET argument, the data will target that
+    node only.
+    """
+    certname = request.args.get('certname')
+    result = get_or_abort(
+        get_daily_reports_chart,
+        db=puppetdb,
+        env=env,
+        days_number=app.config['DAILY_REPORTS_CHART_DAYS'],
+        certname=certname,
+    )
+    return jsonify(result=result)

puppetboard/dailychart.py

@@ -0,0 +1,78 @@
+from datetime import datetime, timedelta
+from pypuppetdb.utils import UTC
+from pypuppetdb.QueryBuilder import (
+    ExtractOperator, FunctionOperator, AndOperator,
+    GreaterEqualOperator, LessOperator, EqualsOperator,
+)
+
+DATE_FORMAT = "%Y-%m-%d"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+def _iter_dates(days_number):
+    """Return a list of datetime pairs AB, BC, CD, ... that represent the
+       24hs time ranges of today (until this midnight) and the
+       previous days.
+    """
+    one_day = timedelta(days=1)
+    today = datetime.utcnow().replace(hour=0, minute=0, second=0,
+                                      microsecond=0, tzinfo=UTC())
+    days_list = list(today + one_day * (1 - i) for i in range(days_number + 1))
+    return zip(days_list[1:], days_list)
+
+
+def _build_query(env, start, end, certname=None):
+    """Build a extract query with optional certname and environment."""
+    query = ExtractOperator()
+    query.add_field(FunctionOperator('count'))
+    query.add_field('status')
+    subquery = AndOperator()
+    subquery.add(GreaterEqualOperator('start_time', start))
+    subquery.add(LessOperator('start_time', end))
+    if certname is not None:
+        subquery.add(EqualsOperator('certname', certname))
+    if env != '*':
+        subquery.add(EqualsOperator('environment', env))
+    query.add_query(subquery)
+    query.add_group_by("status")
+    return query
+
+
+def _format_report_data(day, query_output):
+    """Format the output of the query to a simpler dict."""
+    result = {'day': day, 'changed': 0, 'unchanged': 0, 'failed': 0}
+    for out in query_output:
+        if out['status'] == 'changed':
+            result['changed'] = out['count']
+        elif out['status'] == 'unchanged':
+            result['unchanged'] = out['count']
+        elif out['status'] == 'failed':
+            result['failed'] = out['count']
+    return result
+
+
+def get_daily_reports_chart(db, env, days_number, certname=None):
+    """Return the sum of each report status (changed, unchanged, failed)
+       per day, for today and the previous N days.
+
+    This information is used to present a chart.
+
+    :param db: The puppetdb.
+    :param env: Sum up the reports in this environment.
+    :param days_number: How many days to sum, including today.
+    :param certname: If certname is passed, only the reports of that
+    certname will be added.  If certname is not passed, all reports in
+    the database will be considered.
+    """
+    result = []
+    for start, end in reversed(_iter_dates(days_number)):
+        query = _build_query(
+            env=env,
+            start=start.strftime(DATETIME_FORMAT),
+            end=end.strftime(DATETIME_FORMAT),
+            certname=certname,
+        )
+        day = start.strftime(DATE_FORMAT)
+        output = db._query('reports', query=query)
+        result.append(_format_report_data(day, output))
+    return result

puppetboard/default_settings.py

@@ -38,3 +38,5 @@ INVENTORY_FACTS = [('Hostname', 'fqdn'),
                    ('Kernel Version', 'kernelrelease'),
                    ('Puppet Version', 'puppetversion'), ]
 REFRESH_RATE = 30
+DAILY_REPORTS_CHART_ENABLED = True
+DAILY_REPORTS_CHART_DAYS = 8

puppetboard/static/css/puppetboard.css

@@ -194,3 +194,7 @@ th.tablesorter-headerDesc::after {
   margin-right: -50%;
   transform: translate(-50%, -50%)
 }
+
+#dailyReportsChartContainer {
+  height: 160px;
+}

puppetboard/static/js/dailychart.js

@@ -0,0 +1,39 @@
+jQuery(function ($) {
+  function generateChart(el) {
+    var url = "/daily_reports_chart.json";
+    var certname = $(el).attr('data-certname');
+    if (typeof certname !== typeof undefined && certname !== false) {
+      url = url + "?certname=" + certname;
+    }
+    d3.json(url, function(data) {
+      var chart = c3.generate({
+        bindto: '#dailyReportsChart',
+        data: {
+          type: 'bar',
+          json: data['result'],
+          keys: {
+            x: 'day',
+            value: ['failed', 'changed', 'unchanged'],
+          },
+          groups: [
+            ['failed', 'changed', 'unchanged']
+          ],
+          colors: {  // Must match CSS colors
+            'failed':'#AA4643',
+            'changed':'#4572A7',
+            'unchanged':'#89A54E',
+          }
+        },
+        size: {
+          height: 160
+        },
+        axis: {
+          x: {
+            type: 'category'
+          }
+        }
+      });
+    });
+  }
+  generateChart($("#dailyReportsChart"));
+});

puppetboard/templates/index.html

@@ -1,5 +1,10 @@
 {% extends 'layout.html' %}
 {% import '_macros.html' as macros %}
+{% block head %}
+{% if config.DAILY_REPORTS_CHART_ENABLED %}
+<link href="{{ url_for('static', filename='css/c3.min.css') }}" rel="stylesheet">
+{% endif %}
+{% endblock head %}
 {% block content %}
 {% if config.REFRESH_RATE > 0 %}
 <meta http-equiv="refresh" content="{{config.REFRESH_RATE}}">
@@ -59,6 +64,11 @@
       <span>Avg. resources/node</span>
     </div>
   </div>
+  {% if config.DAILY_REPORTS_CHART_ENABLED %}
+  <div id="dailyReportsChartContainer" class="one column row">
+    <div id="dailyReportsChart"></div>
+  </div>
+  {% endif %}
   <div class="ui divider"></div>
   <div class="one column row">
     <div class="column">
@@ -114,3 +124,10 @@
   </div>
 </div>
 {% endblock content %}
+{% if config.DAILY_REPORTS_CHART_ENABLED %}
+{% block script %}
+<script src="{{url_for('static', filename='js/d3.min.js')}}"></script>
+<script src="{{url_for('static', filename='js/c3.min.js')}}"></script>
+<script src="{{url_for('static', filename='js/dailychart.js')}}"></script>
+{% endblock script %}
+{% endif %}

puppetboard/templates/layout.html

@@ -17,6 +17,7 @@
     {% endif %}
     <link href="{{ url_for('static', filename='Semantic-UI-2.1.8/semantic.min.css') }}" rel="stylesheet">
     <link href="{{ url_for('static', filename='css/puppetboard.css') }}" rel="stylesheet">
+    {% block head %} {% endblock head %}
   </head>
 
   <body>

puppetboard/templates/node.html

@@ -1,5 +1,10 @@
 {% extends 'layout.html' %}
 {% import '_macros.html' as macros %}
+{% block head %}
+{% if config.DAILY_REPORTS_CHART_ENABLED %}
+<link href="{{ url_for('static', filename='css/c3.min.css') }}" rel="stylesheet">
+{% endif %}
+{% endblock head %}
 {% block content %}
 <div class='ui two column grid'>
   <div class='column'>
@@ -28,6 +33,11 @@
     </div>
     <div class='row'>
       <h1>Reports</h1>
+  {% if config.DAILY_REPORTS_CHART_ENABLED %}
+  <div id="dailyReportsChartContainer" class="one column row">
+    <div id="dailyReportsChart" data-certname="{{node.name}}"></div>
+  </div>
+  {% endif %}
       {{ macros.reports_table(reports, reports_count, report_event_counts, condensed=True, hash_truncate=True, show_conf_col=False, show_agent_col=False, show_host_col=False, current_env=current_env)}}
       <a href="{{url_for('reports_node', node_name=node.name)}}">Show All</a>
     </div>
@@ -38,3 +48,10 @@
   </div>
 </div>
 {% endblock content %}
+{% if config.DAILY_REPORTS_CHART_ENABLED %}
+{% block script %}
+<script src="{{url_for('static', filename='js/d3.min.js')}}"></script>
+<script src="{{url_for('static', filename='js/c3.min.js')}}"></script>
+<script src="{{url_for('static', filename='js/dailychart.js')}}"></script>
+{% endblock script %}
+{% endif %}