More Easily View All Reports (#245)

* puppetboard/app.py: Refactoring the report event counts.

Writing the report event counts code in the node(), reports() and
reports_node() functions to iterate through report.events() instead
of querying the event counts endpoint for each report.

This solution is heavier than the original because we have to query
for all full event objects for each report and iterate through them
to interpret their statuses. I originally wanted to replace the
report.events() function with an events variable for the Report object
but that turned out to be not technically possible presently because
the report extended events' timestamps for in a different format which
python can't interpret. Specifically the timezone values contain ':',
there is no Python 2.x documentation that states timezones containing
colons is supported.

This does, however, lighten our dependency on the event-counts endpoint
which is marked as experimental. Which means that it may change or be
removed in a future release.

This also resolves a silent bug which may or may not include environment
filters on the event-counts queries. report.events() searches the Events
endpoint based on the report hash which eliminates the possibility of
mistaken relationships.

* puppetboard/app.py: Replacing url_for_* functions with a single url_for_field()

The url_for_pagination and url_for_environments functions only worked
with a single, fixed request argument, 'page' and 'env' respectively.
The new url_for_field() excepts 2 arguments, field: the name of the
argument to update, and value: its intended value.

Should consider adding a url_for_fields() function that accepts a
dict argument and updates all the request arguments using a dict.update().
There is currently no requirement for it so it will remain in the backlog.

* puppetboard/templates/reports.html: Adding a dropdown menu to limit the report count

This fixes https://github.com/voxpupuli/puppetboard/issues/202

This new dropdown allows users to select their desired number of reports
on the reports() and reports_node() pages. The available options are
app.config['REPORTS_COUNT'], 25, 50, 100 or All. The default value
is determined by the REPORTS_COUNT configuration value.

Had to modify url_for_field() to merge the request args to the view args
in order to generate the links that include the limit query string.

puppetboard/app.py

@@ -57,14 +57,10 @@ def stream_template(template_name, **context):
     rv.enable_buffering(5)
     return rv
 
-def url_for_pagination(page):
+def url_for_field(field, value):
     args = request.view_args.copy()
-    args['page'] = page
-    return url_for(request.endpoint, **args)
-
-def url_for_environments(env):
-    args = request.view_args.copy()
-    args['env'] = env
+    args.update(request.args.copy())
+    args[field] = value
     return url_for(request.endpoint, **args)
 
 def environments():
@@ -80,8 +76,7 @@ def check_env(env, envs):
     if env != '*' and env not in envs:
         abort(404)
 
-app.jinja_env.globals['url_for_pagination'] = url_for_pagination
-app.jinja_env.globals['url_for_environments'] = url_for_environments
+app.jinja_env.globals['url_for_field'] = url_for_field
 
 @app.context_processor
 def utility_processor():
@@ -372,17 +367,29 @@ def node(env, node_name):
     report_event_counts = {}
 
     for report in reports_events:
-        counts = get_or_abort(puppetdb.event_counts,
-            query='["and", ["=", "environment", "{0}"],' \
-                '["=", "certname", "{1}"], ["=", "report", "{2}"]]'.format(
-                    env,
-                    node_name,
-                    report.hash_),
-            summarize_by="certname")
-        try:
-            report_event_counts[report.hash_] = counts[0]
-        except IndexError:
-            report_event_counts[report.hash_] = {}
+        report_event_counts[report.hash_] = {}
+
+        for event in report.events():
+            if event.status == 'success':
+                try:
+                    report_event_counts[report.hash_]['successes'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['successes'] = 1
+            elif event.status == 'failure':
+                try:
+                    report_event_counts[report.hash_]['failures'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['failures'] = 1
+            elif event.status == 'noop':
+                try:
+                    report_event_counts[report.hash_]['noops'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['noops'] = 1
+            elif event.status == 'skipped':
+                try:
+                    report_event_counts[report.hash_]['skips'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['skips'] = 1
     return render_template(
         'node.html',
         node=node,
@@ -409,6 +416,7 @@ def reports(env, page):
     """
     envs = environments()
     check_env(env, envs)
+    limit = request.args.get('limit', app.config['REPORTS_COUNT'])
 
     if env == '*':
         reports_query = None
@@ -418,11 +426,16 @@ def reports(env, page):
         total_query = '["extract", [["function", "count"]],'\
             '["and", ["=", "environment", "{0}"]]]'.format(env)
 
+    try:
+        paging_args = {'limit': int(limit)}
+        paging_args['offset'] = int((page-1) * paging_args['limit'])
+    except ValueError:
+        paging_args = {}
+
     reports = get_or_abort(puppetdb.reports,
         query=reports_query,
-        limit=app.config['REPORTS_COUNT'],
-        offset=(page-1) * app.config['REPORTS_COUNT'],
-        order_by='[{"field": "start_time", "order": "desc"}]')
+        order_by='[{"field": "start_time", "order": "desc"}]',
+        **paging_args)
     total = get_or_abort(puppetdb._query,
         'reports',
         query=total_query)
@@ -434,35 +447,38 @@ def reports(env, page):
         abort(404)
 
     for report in reports_events:
-        if env == '*':
-            event_count_query = '["and",' \
-                '["=", "certname", "{0}"],' \
-                '["=", "report", "{1}"]]'.format(
-                    report.node,
-                    report.hash_)
-        else:
-            event_count_query = '["and",' \
-                '["=", "environment", "{0}"],' \
-                '["=", "certname", "{1}"],' \
-                '["=", "report", "{2}"]]'.format(
-                    env,
-                    report.node,
-                    report.hash_)
-        counts = get_or_abort(puppetdb.event_counts,
-            query=event_count_query,
-            summarize_by="certname")
-        try:
-            report_event_counts[report.hash_] = counts[0]
-        except IndexError:
-            report_event_counts[report.hash_] = {}
+        report_event_counts[report.hash_] = {}
+
+        for event in report.events():
+            if event.status == 'success':
+                try:
+                    report_event_counts[report.hash_]['successes'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['successes'] = 1
+            elif event.status == 'failure':
+                try:
+                    report_event_counts[report.hash_]['failures'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['failures'] = 1
+            elif event.status == 'noop':
+                try:
+                    report_event_counts[report.hash_]['noops'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['noops'] = 1
+            elif event.status == 'skipped':
+                try:
+                    report_event_counts[report.hash_]['skips'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['skips'] = 1
     return Response(stream_with_context(stream_template(
         'reports.html',
         reports=yield_or_stop(reports),
         reports_count=app.config['REPORTS_COUNT'],
         report_event_counts=report_event_counts,
-        pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
+        pagination=Pagination(page, paging_args.get('limit', total), total),
         envs=envs,
-        current_env=env)))
+        current_env=env,
+        limit=paging_args.get('limit', total))))
 
 
 @app.route('/reports/<node_name>/', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1})
@@ -484,11 +500,17 @@ def reports_node(env, node_name, page):
     check_env(env, envs)
 
     if env == '*':
-        query = '["=", "certname", "{0}"]]'.format(node_name)
+        query = '["=", "certname", "{0}"]'.format(node_name)
+        total_query = '["extract", [["function", "count"]],'\
+            '["=", "certname", "{0}"]'.format(node_name)
     else:
         query='["and",' \
             '["=", "environment", "{0}"],' \
-            '["=", "certname", "{1}"]]'.format(env, node_name),
+            '["=", "certname", "{1}"]]'.format(env, node_name)
+        total_query = '["extract", [["function", "count"]],' \
+            '["and",' \
+            '["=", "environment", "{0}"],' \
+            '["=", "certname", "{1}"]]]'.format(env, node_name)
 
     reports = get_or_abort(puppetdb.reports,
         query=query,
@@ -497,10 +519,7 @@ def reports_node(env, node_name, page):
         order_by='[{"field": "start_time", "order": "desc"}]')
     total = get_or_abort(puppetdb._query,
         'reports',
-        query='["extract", [["function", "count"]],' \
-            '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"]]]'.format(
-            env,
-            node_name))
+        query=total_query)
     total = total[0]['count']
     reports, reports_events = tee(reports)
     report_event_counts = {}
@@ -509,27 +528,29 @@ def reports_node(env, node_name, page):
         abort(404)
 
     for report in reports_events:
-        if env == '*':
-            event_count_query = '["and",' \
-                '["=", "certname", "{0}"],' \
-                '["=", "report", "{1}"]]'.format(
-                    report.node,
-                    report.hash_)
-        else:
-            event_count_query = '["and",' \
-                '["=", "environment", "{0}"],' \
-                '["=", "certname", "{1}"],' \
-                '["=", "report", "{2}"]]'.format(
-                    env,
-                    report.node,
-                    report.hash_)
-        counts = get_or_abort(puppetdb.event_counts,
-            query=event_count_query,
-            summarize_by="certname")
-        try:
-            report_event_counts[report.hash_] = counts[0]
-        except IndexError:
-            report_event_counts[report.hash_] = {}
+        report_event_counts[report.hash_] = {}
+
+        for event in report.events():
+            if event.status == 'success':
+                try:
+                    report_event_counts[report.hash_]['successes'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['successes'] = 1
+            elif event.status == 'failure':
+                try:
+                    report_event_counts[report.hash_]['failures'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['failures'] = 1
+            elif event.status == 'noop':
+                try:
+                    report_event_counts[report.hash_]['noops'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['noops'] = 1
+            elif event.status == 'skipped':
+                try:
+                    report_event_counts[report.hash_]['skips'] += 1
+                except KeyError:
+                    report_event_counts[report.hash_]['skips'] = 1
     return render_template(
         'reports.html',
         reports=reports,

puppetboard/templates/_macros.html

@@ -176,13 +176,13 @@
 {% macro render_pagination(pagination) -%}
   <div class="ui pagination menu">
   {% if pagination.has_prev %}
-    <a class="item" href="{{url_for_pagination(1)}}">&laquo; First</a>
-    <a class="item" href="{{url_for_pagination(pagination.page - 1)}}">Prev</a>
+    <a class="item" href="{{url_for_field('page', 1)}}">&laquo; First</a>
+    <a class="item" href="{{url_for_field('page', pagination.page - 1)}}">Prev</a>
   {% endif %}
   {% for page in pagination.iter_pages() %}
     {% if page %}
       {% if page != pagination.page %}
-        <a class="item" href="{{url_for_pagination(page)}}">{{page}}</a>
+        <a class="item" href="{{url_for_field('page', page)}}">{{page}}</a>
       {% else %}
         <a class="active item">{{page}}</a>
       {% endif %}
@@ -191,8 +191,8 @@
     {% endif %}
   {% endfor %}
   {% if pagination.has_next %}
-    <a class="item" href="{{url_for_pagination(pagination.page + 1)}}">Next</a>
-    <a class="item" href="{{url_for_pagination(pagination.pages)}}">Last &raquo;</a>
+    <a class="item" href="{{url_for_field('page', pagination.page + 1)}}">Next</a>
+    <a class="item" href="{{url_for_field('page', pagination.pages)}}">Last &raquo;</a>
   {% endif %}
   </div>
 {% endmacro %}

puppetboard/templates/layout.html

@@ -42,9 +42,9 @@
         Environments
         <i class="dropdown icon"></i>
         <div class="menu">
-            <a class="{% if '*' == current_env %}active {% endif %}item" href="{{url_for_environments('*')}}">All environments</a>
+            <a class="{% if '*' == current_env %}active {% endif %}item" href="{{url_for_field('env', '*')}}">All environments</a>
           {% for env in envs %}
-            <a class="{% if env == current_env %}active {% endif %}item" href="{{url_for_environments(env)}}">{{env}}</a>
+            <a class="{% if env == current_env %}active {% endif %}item" href="{{url_for_field('env', env)}}">{{env}}</a>
           {% endfor %}
         </div>
       </div>

puppetboard/templates/node.html

@@ -29,6 +29,7 @@
     <div class='row'>
       <h1>Reports</h1>
       {{ macros.reports_table(reports, reports_count, report_event_counts, condensed=True, hash_truncate=True, show_conf_col=False, show_agent_col=False, show_host_col=False, current_env=current_env)}}
+      <a href="{{url_for('reports_node', node_name=node.name)}}">Show All</a>
     </div>
   </div>
   <div class='column'>

puppetboard/templates/reports.html

@@ -3,4 +3,15 @@
 {% block content %}
 {{ macros.reports_table(reports, reports_count, report_event_counts, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True, show_search_bar=True, searchable=True, current_env=current_env)}}
 {{ macros.render_pagination(pagination)}}
+<div class="ui dropdown" style="float:right;">
+  <div class="text">Limit</div>
+  <i class="dropdown icon"></i>
+  <div class="menu">
+    <a class="{% if limit == config.REPORTS_COUNT %}active {% endif %}item" href="{{url_for_field('limit', config.REPORTS_COUNT)}}">{{config.REPORTS_COUNT}}</a>
+    <a class="{% if limit == 25 %}active {% endif %}item" href="{{url_for_field('limit', 25)}}">25</a>
+    <a class="{% if limit == 50 %}active {% endif %}item" href="{{url_for_field('limit', 50)}}">50</a>
+    <a class="{% if limit == 100 %}active {% endif %}item" href="{{url_for_field('limit', 100)}}">100</a>
+    <a class="{% if limit == '*' %}active {% endif %}item" href="{{url_for_field('limit', '*')}}">All</a>
+  </div>
+</div>
 {% endblock content %}