From b7cd58ac2c1b1c1f1690e2e7e0c41eabc6e2c1e5 Mon Sep 17 00:00:00 2001 From: Corey Hammerton Date: Mon, 6 Jun 2016 20:09:07 -0400 Subject: [PATCH] More Easily View All Reports (#245) * puppetboard/app.py: Refactoring the report event counts. Writing the report event counts code in the node(), reports() and reports_node() functions to iterate through report.events() instead of querying the event counts endpoint for each report. This solution is heavier than the original because we have to query for all full event objects for each report and iterate through them to interpret their statuses. I originally wanted to replace the report.events() function with an events variable for the Report object but that turned out to be not technically possible presently because the report extended events' timestamps for in a different format which python can't interpret. Specifically the timezone values contain ':', there is no Python 2.x documentation that states timezones containing colons is supported. This does, however, lighten our dependency on the event-counts endpoint which is marked as experimental. Which means that it may change or be removed in a future release. This also resolves a silent bug which may or may not include environment filters on the event-counts queries. report.events() searches the Events endpoint based on the report hash which eliminates the possibility of mistaken relationships. * puppetboard/app.py: Replacing url_for_* functions with a single url_for_field() The url_for_pagination and url_for_environments functions only worked with a single, fixed request argument, 'page' and 'env' respectively. The new url_for_field() excepts 2 arguments, field: the name of the argument to update, and value: its intended value. Should consider adding a url_for_fields() function that accepts a dict argument and updates all the request arguments using a dict.update(). There is currently no requirement for it so it will remain in the backlog. * puppetboard/templates/reports.html: Adding a dropdown menu to limit the report count This fixes https://github.com/voxpupuli/puppetboard/issues/202 This new dropdown allows users to select their desired number of reports on the reports() and reports_node() pages. The available options are app.config['REPORTS_COUNT'], 25, 50, 100 or All. The default value is determined by the REPORTS_COUNT configuration value. Had to modify url_for_field() to merge the request args to the view args in order to generate the links that include the limit query string. --- puppetboard/app.py | 167 ++++++++++++++++------------- puppetboard/templates/_macros.html | 10 +- puppetboard/templates/layout.html | 4 +- puppetboard/templates/node.html | 1 + puppetboard/templates/reports.html | 11 ++ 5 files changed, 113 insertions(+), 80 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index a65e4a7..dbbe05d 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -57,14 +57,10 @@ def stream_template(template_name, **context): rv.enable_buffering(5) return rv -def url_for_pagination(page): +def url_for_field(field, value): args = request.view_args.copy() - args['page'] = page - return url_for(request.endpoint, **args) - -def url_for_environments(env): - args = request.view_args.copy() - args['env'] = env + args.update(request.args.copy()) + args[field] = value return url_for(request.endpoint, **args) def environments(): @@ -80,8 +76,7 @@ def check_env(env, envs): if env != '*' and env not in envs: abort(404) -app.jinja_env.globals['url_for_pagination'] = url_for_pagination -app.jinja_env.globals['url_for_environments'] = url_for_environments +app.jinja_env.globals['url_for_field'] = url_for_field @app.context_processor def utility_processor(): @@ -372,17 +367,29 @@ def node(env, node_name): report_event_counts = {} for report in reports_events: - counts = get_or_abort(puppetdb.event_counts, - query='["and", ["=", "environment", "{0}"],' \ - '["=", "certname", "{1}"], ["=", "report", "{2}"]]'.format( - env, - node_name, - report.hash_), - summarize_by="certname") - try: - report_event_counts[report.hash_] = counts[0] - except IndexError: - report_event_counts[report.hash_] = {} + report_event_counts[report.hash_] = {} + + for event in report.events(): + if event.status == 'success': + try: + report_event_counts[report.hash_]['successes'] += 1 + except KeyError: + report_event_counts[report.hash_]['successes'] = 1 + elif event.status == 'failure': + try: + report_event_counts[report.hash_]['failures'] += 1 + except KeyError: + report_event_counts[report.hash_]['failures'] = 1 + elif event.status == 'noop': + try: + report_event_counts[report.hash_]['noops'] += 1 + except KeyError: + report_event_counts[report.hash_]['noops'] = 1 + elif event.status == 'skipped': + try: + report_event_counts[report.hash_]['skips'] += 1 + except KeyError: + report_event_counts[report.hash_]['skips'] = 1 return render_template( 'node.html', node=node, @@ -409,6 +416,7 @@ def reports(env, page): """ envs = environments() check_env(env, envs) + limit = request.args.get('limit', app.config['REPORTS_COUNT']) if env == '*': reports_query = None @@ -418,11 +426,16 @@ def reports(env, page): total_query = '["extract", [["function", "count"]],'\ '["and", ["=", "environment", "{0}"]]]'.format(env) + try: + paging_args = {'limit': int(limit)} + paging_args['offset'] = int((page-1) * paging_args['limit']) + except ValueError: + paging_args = {} + reports = get_or_abort(puppetdb.reports, query=reports_query, - limit=app.config['REPORTS_COUNT'], - offset=(page-1) * app.config['REPORTS_COUNT'], - order_by='[{"field": "start_time", "order": "desc"}]') + order_by='[{"field": "start_time", "order": "desc"}]', + **paging_args) total = get_or_abort(puppetdb._query, 'reports', query=total_query) @@ -434,35 +447,38 @@ def reports(env, page): abort(404) for report in reports_events: - if env == '*': - event_count_query = '["and",' \ - '["=", "certname", "{0}"],' \ - '["=", "report", "{1}"]]'.format( - report.node, - report.hash_) - else: - event_count_query = '["and",' \ - '["=", "environment", "{0}"],' \ - '["=", "certname", "{1}"],' \ - '["=", "report", "{2}"]]'.format( - env, - report.node, - report.hash_) - counts = get_or_abort(puppetdb.event_counts, - query=event_count_query, - summarize_by="certname") - try: - report_event_counts[report.hash_] = counts[0] - except IndexError: - report_event_counts[report.hash_] = {} + report_event_counts[report.hash_] = {} + + for event in report.events(): + if event.status == 'success': + try: + report_event_counts[report.hash_]['successes'] += 1 + except KeyError: + report_event_counts[report.hash_]['successes'] = 1 + elif event.status == 'failure': + try: + report_event_counts[report.hash_]['failures'] += 1 + except KeyError: + report_event_counts[report.hash_]['failures'] = 1 + elif event.status == 'noop': + try: + report_event_counts[report.hash_]['noops'] += 1 + except KeyError: + report_event_counts[report.hash_]['noops'] = 1 + elif event.status == 'skipped': + try: + report_event_counts[report.hash_]['skips'] += 1 + except KeyError: + report_event_counts[report.hash_]['skips'] = 1 return Response(stream_with_context(stream_template( 'reports.html', reports=yield_or_stop(reports), reports_count=app.config['REPORTS_COUNT'], report_event_counts=report_event_counts, - pagination=Pagination(page, app.config['REPORTS_COUNT'], total), + pagination=Pagination(page, paging_args.get('limit', total), total), envs=envs, - current_env=env))) + current_env=env, + limit=paging_args.get('limit', total)))) @app.route('/reports//', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1}) @@ -484,11 +500,17 @@ def reports_node(env, node_name, page): check_env(env, envs) if env == '*': - query = '["=", "certname", "{0}"]]'.format(node_name) + query = '["=", "certname", "{0}"]'.format(node_name) + total_query = '["extract", [["function", "count"]],'\ + '["=", "certname", "{0}"]'.format(node_name) else: query='["and",' \ '["=", "environment", "{0}"],' \ - '["=", "certname", "{1}"]]'.format(env, node_name), + '["=", "certname", "{1}"]]'.format(env, node_name) + total_query = '["extract", [["function", "count"]],' \ + '["and",' \ + '["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"]]]'.format(env, node_name) reports = get_or_abort(puppetdb.reports, query=query, @@ -497,10 +519,7 @@ def reports_node(env, node_name, page): order_by='[{"field": "start_time", "order": "desc"}]') total = get_or_abort(puppetdb._query, 'reports', - query='["extract", [["function", "count"]],' \ - '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"]]]'.format( - env, - node_name)) + query=total_query) total = total[0]['count'] reports, reports_events = tee(reports) report_event_counts = {} @@ -509,27 +528,29 @@ def reports_node(env, node_name, page): abort(404) for report in reports_events: - if env == '*': - event_count_query = '["and",' \ - '["=", "certname", "{0}"],' \ - '["=", "report", "{1}"]]'.format( - report.node, - report.hash_) - else: - event_count_query = '["and",' \ - '["=", "environment", "{0}"],' \ - '["=", "certname", "{1}"],' \ - '["=", "report", "{2}"]]'.format( - env, - report.node, - report.hash_) - counts = get_or_abort(puppetdb.event_counts, - query=event_count_query, - summarize_by="certname") - try: - report_event_counts[report.hash_] = counts[0] - except IndexError: - report_event_counts[report.hash_] = {} + report_event_counts[report.hash_] = {} + + for event in report.events(): + if event.status == 'success': + try: + report_event_counts[report.hash_]['successes'] += 1 + except KeyError: + report_event_counts[report.hash_]['successes'] = 1 + elif event.status == 'failure': + try: + report_event_counts[report.hash_]['failures'] += 1 + except KeyError: + report_event_counts[report.hash_]['failures'] = 1 + elif event.status == 'noop': + try: + report_event_counts[report.hash_]['noops'] += 1 + except KeyError: + report_event_counts[report.hash_]['noops'] = 1 + elif event.status == 'skipped': + try: + report_event_counts[report.hash_]['skips'] += 1 + except KeyError: + report_event_counts[report.hash_]['skips'] = 1 return render_template( 'reports.html', reports=reports, diff --git a/puppetboard/templates/_macros.html b/puppetboard/templates/_macros.html index 6a0669e..33e0eb0 100644 --- a/puppetboard/templates/_macros.html +++ b/puppetboard/templates/_macros.html @@ -176,13 +176,13 @@ {% macro render_pagination(pagination) -%} {% endmacro %} diff --git a/puppetboard/templates/layout.html b/puppetboard/templates/layout.html index a465103..5798d6e 100644 --- a/puppetboard/templates/layout.html +++ b/puppetboard/templates/layout.html @@ -42,9 +42,9 @@ Environments
- All environments + All environments {% for env in envs %} - {{env}} + {{env}} {% endfor %}
diff --git a/puppetboard/templates/node.html b/puppetboard/templates/node.html index b1e9236..f8158e0 100644 --- a/puppetboard/templates/node.html +++ b/puppetboard/templates/node.html @@ -29,6 +29,7 @@

Reports

{{ macros.reports_table(reports, reports_count, report_event_counts, condensed=True, hash_truncate=True, show_conf_col=False, show_agent_col=False, show_host_col=False, current_env=current_env)}} + Show All
diff --git a/puppetboard/templates/reports.html b/puppetboard/templates/reports.html index 4d98b32..ccaa9e1 100644 --- a/puppetboard/templates/reports.html +++ b/puppetboard/templates/reports.html @@ -3,4 +3,15 @@ {% block content %} {{ macros.reports_table(reports, reports_count, report_event_counts, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True, show_search_bar=True, searchable=True, current_env=current_env)}} {{ macros.render_pagination(pagination)}} +
+
Limit
+ +
+ {{config.REPORTS_COUNT}} + 25 + 50 + 100 + All +
+
{% endblock content %}