From af05f67428e2cad8e7a12b8f0c1158fd45551c45 Mon Sep 17 00:00:00 2001 From: Corey Hammerton Date: Mon, 26 Oct 2015 21:44:33 -0400 Subject: [PATCH] puppetboard: Adding PuppetDB 3.x support Some excerpts from CHANGELOG.rst include: - Increasing the pypuppetdb requirements from 0.1.x to 0.2.x - The Reports page now lists reports from the reports endpoint instead of a link to a PuppetDB issue with a feature request - Adding a Catalogs page to view either individual node catalogs or compare them against other nodes - New environment awareness adds a new query parameter to all applicable endpoints to filter results based on the current environment. If the default environment 'production' is not available, or any other unavailable environment, the user is redirected to the first known environment. - Adding pagination functionality for reports (for now) based on the value of the REPORTS_COUNT configuration option (used for the limit and the offset calculation). Implementation also makes it possible for other UI enhancements. - Removing the limit_reports function from puppetboard/utils.py since paging parameters are now accepted by the pypuppetdb endpoint functions. - Bumping the version to 0.1.0 --- CHANGELOG.rst | 34 ++ puppetboard/app.py | 649 +++++++++++++++++---- puppetboard/forms.py | 17 +- puppetboard/static/css/puppetboard.css | 2 +- puppetboard/static/js/environments.js | 11 + puppetboard/static/js/tables.js | 4 + puppetboard/templates/_macros.html | 111 +++- puppetboard/templates/catalog.html | 2 +- puppetboard/templates/catalog_compare.html | 87 +++ puppetboard/templates/catalogs.html | 40 ++ puppetboard/templates/fact.html | 4 +- puppetboard/templates/facts.html | 2 +- puppetboard/templates/index.html | 37 +- puppetboard/templates/layout.html | 15 +- puppetboard/templates/metrics.html | 2 +- puppetboard/templates/node.html | 4 +- puppetboard/templates/nodes.html | 31 +- puppetboard/templates/query.html | 2 +- puppetboard/templates/report.html | 74 ++- puppetboard/templates/reports.html | 6 +- puppetboard/templates/reports_node.html | 5 - puppetboard/utils.py | 44 +- setup.py | 6 +- 23 files changed, 967 insertions(+), 222 deletions(-) create mode 100644 puppetboard/static/js/environments.js create mode 100644 puppetboard/templates/catalog_compare.html create mode 100644 puppetboard/templates/catalogs.html delete mode 100644 puppetboard/templates/reports_node.html diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f8a86ee..fc6fcb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,40 @@ Changelog This is the changelog for Puppetboard. +0.1.0 +==== + +* Requires pypuppetdb >= 0.2.0 +* Full support for PuppetDB 3.x +* The first directory location is now a Puppet environment which is filtered + on all supported queries. Users can browse different environments with a + select field in the top NavBar +* Using limit, order_by and offset parameters adding pagaination on the Reports + page (available in the NavBar). Functionality is available to pages that + accept a page attribute. +* The report page now directly queries pypuppetdb to match the report_id + value with the report hash or configuration_version fields. +* Catching and aborting with a 404 if the report and report_latest function + queries do not return a generator object. +* Adding a Catalogs page (similar to the Nodes page) with a form to compare + one node's catalog information with that of another node. +* Updating the Query Endpoints for the Query page. +* Adding to ``templates/_macros.html`` status_counts that shows node/report + status information, like what is avaiable on the index and nodes pages, + available to the reports pages and tables also. +* Showing report logs and metrics in the report page. +* Removing ``limit_reports`` from ``utils.py`` because this helper function + has been replaced by the limit PuppetDB paging function. + +**Known Issues** + +* fact_value pages rendered from JSON valued facts return no results. A more + sophisticated API is required to make use of JSON valued facts (through the + factsets, fact-paths and/or fact-contents endpoints for example) +* Switching environments is done through a JavaScript/jQuery function which + could be a potential vulnerability. A dropdown menu, like those available + in Bootstrap, is the most probable alternative. + 0.0.5 ===== diff --git a/puppetboard/app.py b/puppetboard/app.py index 92ea0e5..855f27d 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -8,6 +8,7 @@ try: except ImportError: from urllib.parse import unquote from datetime import datetime +from itertools import tee from flask import ( Flask, render_template, abort, url_for, @@ -18,10 +19,10 @@ from flask_wtf.csrf import CsrfProtect from pypuppetdb import connect -from puppetboard.forms import QueryForm +from puppetboard.forms import (CatalogForm, QueryForm) from puppetboard.utils import ( get_or_abort, yield_or_stop, - limit_reports, jsonprint + jsonprint, Pagination ) @@ -37,7 +38,6 @@ app.secret_key = app.config['SECRET_KEY'] app.jinja_env.filters['jsonprint'] = jsonprint puppetdb = connect( - api_version=3, host=app.config['PUPPETDB_HOST'], port=app.config['PUPPETDB_PORT'], ssl_verify=app.config['PUPPETDB_SSL_VERIFY'], @@ -59,6 +59,27 @@ def stream_template(template_name, **context): rv.enable_buffering(5) return rv +def url_for_pagination(page): + args = request.view_args.copy() + args['page'] = page + return url_for(request.endpoint, **args) + +def url_for_environments(env): + args = request.view_args.copy() + args['env'] = env + return url_for(request.endpoint, **args) + +def environments(): + envs = get_or_abort(puppetdb.environments) + x = [] + + for env in envs: + x.append(env['name']) + + return x + +app.jinja_env.globals['url_for_pagination'] = url_for_pagination +app.jinja_env.globals['url_for_environments'] = url_for_environments @app.context_processor def utility_processor(): @@ -95,14 +116,23 @@ def server_error(e): return render_template('500.html'), 500 -@app.route('/') -def index(): +@app.route('/', defaults={'env': 'production'}) +@app.route('//') +def index(env): """This view generates the index page and displays a set of metrics and latest reports on nodes fetched from PuppetDB. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + # TODO: Would be great if we could parallelize this somehow, doing these # requests in sequence is rather pointless. - prefix = 'com.puppetlabs.puppetdb.query.population' + prefix = 'puppetlabs.puppetdb.query.population' num_nodes = get_or_abort( puppetdb.metric, "{0}{1}".format(prefix, ':type=default,name=num-nodes')) @@ -118,7 +148,10 @@ def index(): 'avg_resources_node': "{0:10.0f}".format(avg_resources_node['Value']), } - nodes = puppetdb.nodes( + nodes = get_or_abort(puppetdb.nodes, + query='["and", {0}]'.format( + ", ".join('["=", "{0}", "{1}"]'.format(field, env) + for field in ['catalog_environment', 'facts_environment'])), unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True) @@ -150,12 +183,15 @@ def index(): 'index.html', metrics=metrics, nodes=nodes_overview, - stats=stats + stats=stats, + envs=envs, + current_env=env ) -@app.route('/nodes') -def nodes(): +@app.route('/nodes', defaults={'env': 'production'}) +@app.route('//nodes') +def nodes(env): """Fetch all (active) nodes from PuppetDB and stream a table displaying those nodes. @@ -164,9 +200,20 @@ def nodes(): we'll end up with an empty table instead because of how yield_or_stop works. Once pagination is in place we can change this but we'll need to provide a search feature instead. + + :param env: Search for nodes in this (Catalog and Fact) environment + :type env: :obj:`string` """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + status_arg = request.args.get('status', '') nodelist = puppetdb.nodes( + query='["and", {0}]'.format( + ", ".join('["=", "{0}", "{1}"]'.format(field, env) + for field in ['catalog_environment', 'facts_environment'])), unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True) nodes = [] @@ -177,11 +224,15 @@ def nodes(): else: nodes.append(node) return Response(stream_with_context( - stream_template('nodes.html', nodes=nodes))) + stream_template('nodes.html', + nodes=nodes, + envs=envs, + current_env=env))) -@app.route('/inventory') -def inventory(): +@app.route('/inventory', defaults={'env': 'production'}) +@app.route('//inventory') +def inventory(env): """Fetch all (active) nodes from PuppetDB and stream a table displaying those nodes along with a set of facts about them. @@ -190,7 +241,14 @@ def inventory(): we'll end up with an empty table instead because of how yield_or_stop works. Once pagination is in place we can change this but we'll need to provide a search feature instead. + + :param env: Search for facts in this environment + :type env: :obj:`string` """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) fact_desc = [] # a list of fact descriptions to go # in the table header @@ -217,7 +275,8 @@ def inventory(): fact_desc.append(description) fact_names.append(name) - query = '["or", {0}]'.format( + query = '["and", ["=", "environment", "{0}"], ["or", {1}]]'.format( + env, ', '.join('["=", "name", "{0}"]'.format(name) for name in fact_names)) @@ -239,92 +298,270 @@ def inventory(): nodedata[node].append("undef") return Response(stream_with_context( - stream_template('inventory.html', nodedata=nodedata, fact_desc=fact_desc))) + stream_template('inventory.html', + nodedata=nodedata, + fact_desc=fact_desc, + envs=envs, + current_env=env))) -@app.route('/node/') -def node(node_name): +@app.route('/node/', defaults={'env': 'production'}) +@app.route('//node/') +def node(env, node_name): """Display a dashboard for a node showing as much data as we have on that node. This includes facts and reports but not Resources as that is too heavy to do within a single request. + + :param env: Ensure that the node, facts and reports are in this environment + :type env: :obj:`string` """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + node = get_or_abort(puppetdb.node, node_name) facts = node.facts() - reports = limit_reports(node.reports(), app.config['REPORTS_COUNT']) + reports = get_or_abort(puppetdb.reports, + query='["and", ["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"]]'.format(env, node_name), + limit=app.config['REPORTS_COUNT'], + order_by='[{"field": "start_time", "order": "desc"}]') + reports, reports_events = tee(reports) + report_event_counts = {} + + for report in reports_events: + counts = get_or_abort(puppetdb.event_counts, + query='["and", ["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"], ["=", "report", "{2}"]]'.format( + env, + node_name, + report.hash_), + summarize_by="certname") + try: + report_event_counts[report.hash_] = counts[0] + except IndexError: + report_event_counts[report.hash_] = {} return render_template( 'node.html', node=node, facts=yield_or_stop(facts), reports=yield_or_stop(reports), - reports_count=app.config['REPORTS_COUNT']) + reports_count=app.config['REPORTS_COUNT'], + report_event_counts=report_event_counts, + envs=envs, + current_env=env) -@app.route('/reports') -def reports(): - """Doesn't do much yet but is meant to show something like the reports of - the last half our, something like that.""" - return render_template('reports.html') +@app.route('/reports/', defaults={'env': 'production', 'page': 1}) +@app.route('//reports/', defaults={'page': 1}) +@app.route('//reports/page/') +def reports(env, page): + """Displays a list of reports and status from all nodes, retreived using the + reports endpoint, sorted by start_time. - -@app.route('/reports/') -def reports_node(node_name): - """Fetches all reports for a node and processes them eventually rendering - a table displaying those reports.""" - reports = limit_reports( - yield_or_stop( - puppetdb.reports('["=", "certname", "{0}"]'.format(node_name))), - app.config['REPORTS_COUNT']) - return render_template( - 'reports_node.html', - reports=reports, - nodename=node_name, - reports_count=app.config['REPORTS_COUNT']) - - -@app.route('/report/latest/') -def report_latest(node_name): - """Redirect to the latest report of a given node. This is a workaround - as long as PuppetDB can't filter reports for latest-report? field. This - feature has been requested: https://tickets.puppetlabs.com/browse/PDB-203 + :param env: Search for all reports in this environment + :type env: :obj:`string` + :param page: Calculates the offset of the query based on the report count + and this value + :type page: :obj:`int` """ - reports = get_or_abort(puppetdb._query, 'reports', - query='["=","certname","{0}"]'.format(node_name), - limit=1) - if len(reports) > 0: - report = reports[0]['hash'] - return redirect( - url_for('report', node_name=node_name, report_id=report)) - else: + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + reports = get_or_abort(puppetdb.reports, + query='["=", "environment", "{0}"]'.format(env), + limit=app.config['REPORTS_COUNT'], + offset=(page-1) * app.config['REPORTS_COUNT'], + order_by='[{"field": "start_time", "order": "desc"}]') + total = get_or_abort(puppetdb._query, + 'reports', + query='["extract", [["function", "count"]],'\ + '["and", ["=", "environment", "{0}"], ["~", "certname", ""]]]'.format( + env)) + total = total[0]['count'] + reports, reports_events = tee(reports) + report_event_counts = {} + + if total == 0 and page != 1: abort(404) + for report in reports_events: + counts = get_or_abort(puppetdb.event_counts, + query='["and",' \ + '["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"],' \ + '["=", "report", "{2}"]]'.format( + env, + report.node, + report.hash_), + summarize_by="certname") + try: + report_event_counts[report.hash_] = counts[0] + except IndexError: + report_event_counts[report.hash_] = {} + return Response(stream_with_context(stream_template( + 'reports.html', + reports=yield_or_stop(reports), + reports_count=app.config['REPORTS_COUNT'], + report_event_counts=report_event_counts, + pagination=Pagination(page, app.config['REPORTS_COUNT'], total), + envs=envs, + current_env=env))) -@app.route('/report//') -def report(node_name, report_id): + +@app.route('/reports//', defaults={'env': 'production', 'page': 1}) +@app.route('//reports/', defaults={'page': 1}) +@app.route('//reports//page/') +def reports_node(env, node_name, page): + """Fetches all reports for a node and processes them eventually rendering + a table displaying those reports. + + :param env: Search for reports in this environment + :type env: :obj:`string` + :param node_name: Find the reports whose certname match this value + :type node_name: :obj:`string` + :param page: Calculates the offset of the query based on the report count + and this value + :type page: :obj:`int` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + reports = get_or_abort(puppetdb.reports, + query='["and",' \ + '["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"]]'.format(env, node_name), + limit=app.config['REPORTS_COUNT'], + offset=(page-1) * app.config['REPORTS_COUNT'], + order_by='[{"field": "start_time", "order": "desc"}]') + total = get_or_abort(puppetdb._query, + 'reports', + query='["extract", [["function", "count"]],' \ + '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"]]]'.format( + env, + node_name)) + total = total[0]['count'] + reports, reports_events = tee(reports) + report_event_counts = {} + + if total == 0 and page != 1: + abort(404) + + for report in reports_events: + counts = get_or_abort(puppetdb.event_counts, + query='["and",' \ + '["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"],' \ + '["=", "report", "{2}"]]'.format(env, report.node, report.hash_), + summarize_by="certname") + try: + report_event_counts[report.hash_] = counts[0] + except IndexError: + report_event_counts[report.hash_] = {} + return render_template( + 'reports.html', + reports=reports, + reports_count=app.config['REPORTS_COUNT'], + report_event_counts=report_event_counts, + pagination=Pagination(page, app.config['REPORTS_COUNT'], total), + envs=envs, + current_env=env) + + +@app.route('/report/latest/', defaults={'env': 'production'}) +@app.route('//report/latest/') +def report_latest(env, node_name): + """Redirect to the latest report of a given node. + + :param env: Search for reports in this environment + :type env: :obj:`string` + :param node_name: Find the reports whose certname match this value + :type node_name: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + reports = get_or_abort(puppetdb.reports, + query='["and",' \ + '["=", "environment", "{0}"],' \ + '["=", "certname", "{1}"],' \ + '["=", "latest_report?", true]]'.format( + env, + node_name)) + try: + report = next(reports) + except StopIteration: + abort(404) + + return redirect( + url_for('report', env=env, node_name=node_name, report_id=report.hash_)) + + +@app.route('/report//', defaults={'env': 'production'}) +@app.route('//report//') +def report(env, node_name, report_id): """Displays a single report including all the events associated with that report and their status. The report_id may be the puppetdb's report hash or the configuration_version. This allows for better integration into puppet-hipchat. - """ - reports = puppetdb.reports('["=", "certname", "{0}"]'.format(node_name)) - for report in reports: - if report.hash_ == report_id or report.version == report_id: - events = puppetdb.events('["=", "report", "{0}"]'.format( - report.hash_)) - return render_template( - 'report.html', - report=report, - events=yield_or_stop(events)) - else: + :param env: Search for reports in this environment + :type env: :obj:`string` + :param node_name: Find the reports whose certname match this value + :type node_name: :obj:`string` + :param report_id: The hash or the configuration_version of the desired + report + :type report_id: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + query = '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"],' \ + '["or", ["=", "hash", "{2}"], ["=", "configuration_version", "{2}"]]]'.format( + env, node_name, report_id) + reports = puppetdb.reports(query=query) + + try: + report = next(reports) + except StopIteration: abort(404) + return render_template( + 'report.html', + report=report, + events=yield_or_stop(report.events()), + logs=report.logs, + metrics=report.metrics, + envs=envs, + current_env=env) -@app.route('/facts') -def facts(): + +@app.route('/facts', defaults={'env': 'production'}) +@app.route('//facts') +def facts(env): """Displays an alphabetical list of all facts currently known to - PuppetDB.""" + PuppetDB. + + :param env: Serves no purpose for this function, only for consistency's + sake + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + facts_dict = collections.defaultdict(list) facts = get_or_abort(puppetdb.fact_names) for fact in facts: @@ -334,45 +571,94 @@ def facts(): facts_dict[letter] = letter_list sorted_facts_dict = sorted(facts_dict.items()) - return render_template('facts.html', facts_dict=sorted_facts_dict) + return render_template('facts.html', + facts_dict=sorted_facts_dict, + envs=envs, + current_env=env) -@app.route('/fact/') -def fact(fact): +@app.route('/fact/', defaults={'env': 'production'}) +@app.route('//fact/') +def fact(env, fact): """Fetches the specific fact from PuppetDB and displays its value per - node for which this fact is known.""" + node for which this fact is known. + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + # we can only consume the generator once, lists can be doubly consumed # om nom nom render_graph = False if fact in graph_facts: render_graph = True - localfacts = [f for f in yield_or_stop(puppetdb.facts(name=fact))] + localfacts = [f for f in yield_or_stop(puppetdb.facts( + name=fact, + query='["=", "environment", "{0}"]'.format(env)))] return Response(stream_with_context(stream_template( 'fact.html', name=fact, render_graph=render_graph, - facts=localfacts))) + facts=localfacts, + envs=envs, + current_env=env))) -@app.route('/fact//') -def fact_value(fact, value): - """On asking for fact/value get all nodes with that fact.""" - facts = get_or_abort(puppetdb.facts, fact, value) +@app.route('/fact//', defaults={'env': 'production'}) +@app.route('//fact//') +def fact_value(env, fact, value): + """On asking for fact/value get all nodes with that fact. + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Filter facts whose value is equal to this + :type value: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + facts = get_or_abort(puppetdb.facts, + name=fact, + value=value, + query='["=", "environment", "{0}"]'.format(env)) localfacts = [f for f in yield_or_stop(facts)] return render_template( 'fact.html', name=fact, value=value, - facts=localfacts) + facts=localfacts, + envs=envs, + current_env=env) -@app.route('/query', methods=('GET', 'POST')) -def query(): +@app.route('/query', methods=('GET', 'POST'), defaults={'env': 'production'}) +@app.route('//query', methods=('GET', 'POST')) +def query(env): """Allows to execute raw, user created querries against PuppetDB. This is currently highly experimental and explodes in interesting ways since none of the possible exceptions are being handled just yet. This will return the JSON of the response or a message telling you what whent wrong / - why nothing was returned.""" + why nothing was returned. + + :param env: Serves no purpose for the query data but is required for the + select field in the environment block + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + if app.config['ENABLE_QUERY']: form = QueryForm() if form.validate_on_submit(): @@ -384,36 +670,203 @@ def query(): puppetdb._query, form.endpoints.data, query=query) - return render_template('query.html', form=form, result=result) - return render_template('query.html', form=form) + return render_template('query.html', + form=form, + result=result, + envs=envs, + current_env=env) + return render_template('query.html', + form=form, + envs=envs, + current_env=env) else: log.warn('Access to query interface disabled by administrator..') abort(403) -@app.route('/metrics') -def metrics(): - metrics = get_or_abort(puppetdb._query, 'metrics', path='mbeans') +@app.route('/metrics', defaults={'env': 'production'}) +@app.route('//metrics') +def metrics(env): + """Lists all available metrics that PuppetDB is aware of. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + metrics = get_or_abort(puppetdb._query, 'mbean') for key, value in metrics.items(): - metrics[key] = value.split('/')[3] - return render_template('metrics.html', metrics=sorted(metrics.items())) + metrics[key] = value.split('/')[2] + return render_template('metrics.html', + metrics=sorted(metrics.items()), + envs=envs, + current_env=env) -@app.route('/metric/') -def metric(metric): +@app.route('/metric/', defaults={'env': 'production'}) +@app.route('//metric/') +def metric(env, metric): + """Lists all information about the metric of the given name. + + :param env: While this parameter serves no function purpose it is required + for the environments template block + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + name = unquote(metric) metric = puppetdb.metric(metric) return render_template( 'metric.html', name=name, - metric=sorted(metric.items())) + metric=sorted(metric.items()), + envs=envs, + current_env=env) + +@app.route('/catalogs', defaults={'env': 'production'}) +@app.route('//catalogs') +def catalogs(env): + """Lists all nodes with a compiled catalog. + + :param env: Find the nodes with this catalog_environment value + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) -@app.route('/catalog/') -def catalog_node(node_name): - """Fetches from PuppetDB the compiled catalog of a given node.""" if app.config['ENABLE_CATALOG']: - catalog = puppetdb.catalog(node=node_name) - return render_template('catalog.html', catalog=catalog) + nodenames = [] + catalog_list = [] + nodes = get_or_abort(puppetdb.nodes, + query='["and",' \ + '["=", "catalog_environment", "{0}"],' \ + '["null?", "catalog_timestamp", false]]'.format(env), + with_status=False, + order_by='[{"field": "certname", "order": "asc"}]') + nodes, temp = tee(nodes) + + for node in temp: + nodenames.append(node.name) + + for node in nodes: + table_row = { + 'name': node.name, + 'catalog_timestamp': node.catalog_timestamp + } + + if len(nodenames) > 1: + form = CatalogForm() + + form.compare.data = node.name + form.against.choices = [(x, x) for x in nodenames + if x != node.name] + table_row['form'] = form + else: + table_row['form'] = None + + catalog_list.append(table_row) + + return render_template( + 'catalogs.html', + nodes=catalog_list, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + +@app.route('/catalog/', defaults={'env': 'production'}) +@app.route('//catalog/') +def catalog_node(env, node_name): + """Fetches from PuppetDB the compiled catalog of a given node. + + :param env: Find the catalog with this environment value + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + if app.config['ENABLE_CATALOG']: + catalog = get_or_abort(puppetdb.catalog, + node=node_name) + return render_template('catalog.html', + catalog=catalog, + envs=envs, + current_env=env) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + +@app.route('/catalog/submit', methods=['POST'], defaults={'env': 'production'}) +@app.route('//catalog/submit', methods=['POST']) +def catalog_submit(env): + """Receives the submitted form data from the catalogs page and directs + the users to the comparison page. Directs users back to the catalogs + page if no form submission data is found. + + :param env: This parameter only directs the response page to the right + environment. If this environment does not exist return the use to the + catalogs page with the right environment. + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + if app.config['ENABLE_CATALOG']: + form = CatalogForm(request.form) + + form.against.choices = [(form.against.data, form.against.data)] + if form.validate_on_submit(): + compare = form.compare.data + against = form.against.data + return redirect( + url_for('catalog_compare', + env=env, + compare=compare, + against=against)) + return redirect(url_for('catalogs', env=env)) + else: + log.warn('Access to catalog interface disabled by administrator') + abort(403) + +@app.route('/catalogs/compare/...', defaults={'env': 'production'}) +@app.route('//catalogs/compare/...') +def catalog_compare(env, compare, against): + """Compares the catalog of one node, parameter compare, with that of + with that of another node, parameter against. + + :param env: Ensure that the 2 catalogs are in the same environment + :type env: :obj:`string` + """ + envs = environments() + + if env not in envs: + return redirect(url_for_environments(envs[0])) + + if app.config['ENABLE_CATALOG']: + compare_cat = get_or_abort(puppetdb.catalog, + node=compare) + against_cat = get_or_abort(puppetdb.catalog, + node=against) + + return render_template('catalog_compare.html', + compare=compare_cat, + against=against_cat, + envs=envs, + current_env=env) else: log.warn('Access to catalog interface disabled by administrator') abort(403) diff --git a/puppetboard/forms.py b/puppetboard/forms.py index f4667c3..adb6106 100644 --- a/puppetboard/forms.py +++ b/puppetboard/forms.py @@ -2,7 +2,10 @@ from __future__ import unicode_literals from __future__ import absolute_import from flask.ext.wtf import Form -from wtforms import RadioField, TextAreaField, validators +from wtforms import ( + HiddenField, RadioField, SelectField, + TextAreaField, validators +) class QueryForm(Form): @@ -14,7 +17,17 @@ class QueryForm(Form): ('nodes', 'Nodes'), ('resources', 'Resources'), ('facts', 'Facts'), - ('fact-names', 'Fact Names'), + ('factsets', 'Fact Sets'), + ('fact-paths', 'Fact Paths'), + ('fact-contents', 'Fact Contents'), ('reports', 'Reports'), ('events', 'Events'), + ('catalogs', 'Catalogs'), + ('edges', 'Edges'), + ('environments', 'Environments'), ]) + +class CatalogForm(Form): + """The form used to compare the catalogs of different nodes.""" + compare = HiddenField('compare') + against = SelectField(u'against') diff --git a/puppetboard/static/css/puppetboard.css b/puppetboard/static/css/puppetboard.css index 8c0faf7..05794b4 100644 --- a/puppetboard/static/css/puppetboard.css +++ b/puppetboard/static/css/puppetboard.css @@ -31,7 +31,7 @@ th.tablesorter-headerDesc::after { } .status { - width: 47.5%; + width: 45%; text-align: center; display: block; } diff --git a/puppetboard/static/js/environments.js b/puppetboard/static/js/environments.js new file mode 100644 index 0000000..bc3cca0 --- /dev/null +++ b/puppetboard/static/js/environments.js @@ -0,0 +1,11 @@ +(function () { + var $; + + $ = jQuery; + + $('#switch_env').change(function() { + path = location.pathname.split('/'); + path[1] = $(this).find(':selected').text(); + location.assign(path.join('/')) + }); +}).call(this) diff --git a/puppetboard/static/js/tables.js b/puppetboard/static/js/tables.js index fc85af4..dfe0a05 100644 --- a/puppetboard/static/js/tables.js +++ b/puppetboard/static/js/tables.js @@ -57,6 +57,10 @@ sortList: [[0, 0]] }) + $('.reports').tablesorter({ + sortList: [[0, 0]] + }) + $('input.filter-table').parent('div').removeClass('hide'); $("input.filter-table").on("keyup", function(e) { diff --git a/puppetboard/templates/_macros.html b/puppetboard/templates/_macros.html index df99033..03a7c90 100644 --- a/puppetboard/templates/_macros.html +++ b/puppetboard/templates/_macros.html @@ -1,4 +1,4 @@ -{% macro facts_table(facts, autofocus=False, condensed=False, show_node=False, show_value=True, link_facts=False, margin_top=20, margin_bottom=20) -%} +{% macro facts_table(facts, current_env, autofocus=False, condensed=False, show_node=False, show_value=True, link_facts=False, margin_top=20, margin_bottom=20) -%}
@@ -19,16 +19,24 @@ {% for fact in facts %} {% if show_node %} - {{fact.node}} + {{fact.node}} {% else %} - {{fact.name}} + {{fact.name}} {% endif %} {% if show_value %} {% if link_facts %} - {{fact.value}} + {% if fact.value is mapping %} + 
{{fact.value|jsonprint}}
 + {% else %} + {{fact.value}} + {% endif %} {% else %} - {{fact.value}} + {% if fact.value is mapping %} + 
{{fact.value|jsonprint}}
+ {% else %} + {{fact.value}} + {% endif %} {% endif %} {% endif %} @@ -70,30 +78,40 @@ {%- endmacro %} -{% macro reports_table(reports, nodename, reports_count, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True) -%} +{% macro reports_table(reports, reports_count, report_event_counts, current_env, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True, show_run_col=False, show_full_col=False, show_search_bar=False, searchable=False) -%} +{% if show_search_bar %} +
+ +
+{% endif %}
- Only showing the last {{reports_count}} reports. + Only showing {{reports_count}} reports sorted by Start Time.
- +
+ + {% if show_host_col %} + + {% endif %} + {% if show_run_col %} + {% endif %} + {% if show_full_col %} + {% endif %} {% if show_conf_col %} {% endif %} {% if show_agent_col %} {% endif %} - {% if show_host_col %} - - {% endif %} - + {% for report in reports %} {% if hash_truncate %} {% set rep_hash = "%s…"|format(report.hash_[0:10])|safe %} @@ -105,21 +123,78 @@ {% else %} {% endif %} - + + + {% if show_host_col %} + + {% endif %} + {% if show_run_col %} - - + {% endif %} + {% if show_full_col %} + + {% endif %} {% if show_conf_col %} {% endif %} {% if show_agent_col %} {% endif %} - {% if show_host_col %} - - {% endif %} {% endfor %}
Start timeStatusHostnameRun timeFull reportConfiguration versionAgent versionHostname
{{report.start}}{{report.start}} + {% call status_counts(status=report.status, node_name=report.node, events=report_event_counts[report.hash_], report_hash=report.hash_, current_env=current_env) %}{% endcall %} + {{ report.node }}{{report.run_time}}{{rep_hash}}{{rep_hash}}{{report.version}}{{report.agent_version}}{{ report.node }}
{%- endmacro %} +{% macro status_counts(caller, status, node_name, events, current_env, unreported_time=False, report_hash=False) -%} + + {{status}} + + {% if status == 'unreported' %} + {{ unreported_time }} + {% else %} + {% if events['failures'] %}{{events['failures']}}{% else %}0{% endif%} + {% if events['successes'] %}{{events['successes']}}{% else %}0{% endif%} + {% if events['skips'] %}{{events['skips']}}{% else %}0{% endif%} + {% endif %} +{%- endmacro %} +{% macro render_pagination(pagination) -%} + +{% endmacro %} diff --git a/puppetboard/templates/catalog.html b/puppetboard/templates/catalog.html index 3a43248..11c1e16 100644 --- a/puppetboard/templates/catalog.html +++ b/puppetboard/templates/catalog.html @@ -14,7 +14,7 @@ - {{catalog.node}} + {{catalog.node}} {{catalog.version}} {{catalog.transaction_uuid}} diff --git a/puppetboard/templates/catalog_compare.html b/puppetboard/templates/catalog_compare.html new file mode 100644 index 0000000..705210a --- /dev/null +++ b/puppetboard/templates/catalog_compare.html @@ -0,0 +1,87 @@ +{% extends 'layout.html' %} +{% block content %} + + + + + + + + + + + + + + + + + + + +

Comparing

Against

{{compare.node}}{{against.node}}
+ + + + + + {% for resource in compare.get_resources() %} + + + + {% endfor %} + +
Resources
{{resource.type_}}[{{resource.name}}]
+ 		+ + + + + + {% for resource in against.get_resources() %} + + + + {% endfor %} + +
Resources
{{resource.type_}}[{{resource.name}}]
+
+ + + + + + + + + + {% for edge in compare.get_edges() %} + + + + + + {% endfor %} + +
Edges->Target
{{edge.source}}{{edge.relationship}}{{edge.target}}
+ 		+ + + + + + + + + + {% for edge in against.get_edges() %} + + + + + + {% endfor %} + +
Edge->Target
{{edge.source}}{{edge.relationship}}{{edge.target}}
+
+{% endblock content %} diff --git a/puppetboard/templates/catalogs.html b/puppetboard/templates/catalogs.html new file mode 100644 index 0000000..a84b1e8 --- /dev/null +++ b/puppetboard/templates/catalogs.html @@ -0,0 +1,40 @@ +{% extends 'layout.html' %} +{% import '_macros.html' as macros %} +{% block content %} +
+ +
+ + + + + + + + + + + {% for node in nodes %} + + + + + + + {% endfor %} + +
HostnameCompile TimeCompare With
{{node.name}}{{node.catalog_timestamp}} + {% if node.form %} +
+
+ {{node.form.csrf_token}} +
+ {{node.form.compare}} + {{node.form.against}} + +
+
+
+ {% endif %} +
+{% endblock content %} diff --git a/puppetboard/templates/fact.html b/puppetboard/templates/fact.html index 6120fc0..dbef184 100644 --- a/puppetboard/templates/fact.html +++ b/puppetboard/templates/fact.html @@ -6,8 +6,8 @@ {{macros.facts_graph(facts, autofocus=True, show_node=True, margin_bottom=10)}} {% endif %} {% if value %} -{{macros.facts_table(facts, autofocus=True, show_node=True, show_value=False, margin_bottom=10)}} +{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, show_value=False, margin_bottom=10)}} {% else %} -{{macros.facts_table(facts, autofocus=True, show_node=True, link_facts=True, margin_bottom=10)}} +{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, link_facts=True, margin_bottom=10)}} {% endif %} {% endblock content %} diff --git a/puppetboard/templates/facts.html b/puppetboard/templates/facts.html index b6fe29b..12e8931 100644 --- a/puppetboard/templates/facts.html +++ b/puppetboard/templates/facts.html @@ -8,7 +8,7 @@ {{key}} {% endfor %} diff --git a/puppetboard/templates/index.html b/puppetboard/templates/index.html index 1c3e4b8..7ff87a0 100644 --- a/puppetboard/templates/index.html +++ b/puppetboard/templates/index.html @@ -1,9 +1,10 @@ {% extends 'layout.html' %} +{% import '_macros.html' as macros %} {% block content %}
- +

{{stats['failed']}} {% if stats['failed']== 1 %} node {% else %} nodes {% endif %} @@ -12,7 +13,7 @@ with status failed

- +

{{stats['noop']}} {% if stats['noop']== 1 %} node {% else %} nodes {% endif %} @@ -21,7 +22,7 @@ with status pending

- +

{{stats['changed']}} {% if stats['changed']== 1 %} node {% else %} nodes {% endif %} @@ -30,7 +31,7 @@ with status changed

- +

{{ stats['unreported'] }} {% if stats['unreported']== 1 %} node {% else %} nodes {% endif %} @@ -41,6 +42,7 @@

+ Global Metrics:

{{metrics['num_nodes']}}

@@ -75,40 +77,21 @@ {% if node.status != 'unchanged' %} - - {{node.status}} - - {% if node.status=='unreported'%} - {{ node.unreported_time }} - {% else %} - {% if node.events['failures'] %}{{node.events['failures']}}{% else %}0{% endif%} - {% if node.events['successes'] %}{{node.events['successes']}}{% else %}0{% endif%} - {% if node.events['skips'] %}{{node.events['skips']}}{% else %}0{% endif%} - {% endif %} + {{macros.status_counts(status=node.status, node_name=node.name, events=node.events, unreported_time=node.unreported_time, current_env=current_env)}} - {{ node.name }} + {{ node.name }} {% if node.report_timestamp %} - {{ node.report_timestamp }} + {{ node.report_timestamp }} {% else %} {% endif %} {% if node.report_timestamp %} - + {% endif %} diff --git a/puppetboard/templates/layout.html b/puppetboard/templates/layout.html index f60c8de..eced57a 100644 --- a/puppetboard/templates/layout.html +++ b/puppetboard/templates/layout.html @@ -34,12 +34,22 @@ ('reports', 'Reports'), ('metrics', 'Metrics'), ('inventory', 'Inventory'), + ('catalogs', 'Catalogs'), ('query', 'Query'), ] %} {{ caption }} + href="{{ url_for(endpoint, env=current_env) }}">{{ caption }} {%- endfor %} -
v0.0.5
+ {% if envs|length > 0 %} +
+ +
+ {% endif %} +
v0.1.0
@@ -80,6 +90,7 @@ + {% block script %} {% endblock script %} diff --git a/puppetboard/templates/metrics.html b/puppetboard/templates/metrics.html index cf90e13..33af3c3 100644 --- a/puppetboard/templates/metrics.html +++ b/puppetboard/templates/metrics.html @@ -3,7 +3,7 @@

Metrics

{% endblock content %} diff --git a/puppetboard/templates/node.html b/puppetboard/templates/node.html index b0e83c9..ee8e31f 100644 --- a/puppetboard/templates/node.html +++ b/puppetboard/templates/node.html @@ -28,12 +28,12 @@

Reports

- {{ macros.reports_table(reports, node.name, reports_count, condensed=True, hash_truncate=True, show_conf_col=False, show_agent_col=False, show_host_col=False)}} + {{ macros.reports_table(reports, reports_count, report_event_counts, condensed=True, hash_truncate=True, show_conf_col=False, show_agent_col=False, show_host_col=False, current_env=current_env)}}

Facts

- {{macros.facts_table(facts, link_facts=True, condensed=True)}} + {{macros.facts_table(facts, link_facts=True, condensed=True, current_env=current_env)}}
{% endblock content %} diff --git a/puppetboard/templates/nodes.html b/puppetboard/templates/nodes.html index c657706..bcf715d 100644 --- a/puppetboard/templates/nodes.html +++ b/puppetboard/templates/nodes.html @@ -1,4 +1,5 @@ {% extends 'layout.html' %} +{% import '_macros.html' as macros %} {% block content %}
@@ -17,40 +18,20 @@ {% for node in nodes %} - - {{node.status}} - - {% if node.status=='unreported'%} - {{ node.unreported_time }} - {% else %} - {% if node.events['failures'] %}{{node.events['failures']}}{% else %}0{% endif%} - {% if node.events['successes'] %}{{node.events['successes']}}{% else %}0{% endif%} - {% if node.events['skips'] %}{{node.events['skips']}}{% else %}0{% endif%} - {% endif %} + {{macros.status_counts(status=node.status, node_name=node.name, events=node.events, unreported_time=node.unreported_time, current_env=current_env)}} - {{node.name}} - {{node.catalog_timestamp}} + {{node.name}} + {{node.catalog_timestamp}} {% if node.report_timestamp %} - {{ node.report_timestamp }} + {{ node.report_timestamp }} {% else %} {% endif %} {% if node.report_timestamp %} - - + {% endif %} diff --git a/puppetboard/templates/query.html b/puppetboard/templates/query.html index 86327c3..12d8087 100644 --- a/puppetboard/templates/query.html +++ b/puppetboard/templates/query.html @@ -11,7 +11,7 @@ {% endif %} {% endwith %}
-
+ {{ form.csrf_token }}
{{ form.query(autofocus="autofocus", rows=5, placeholder="Enter your query: [\"=\", \"certname\", \"hostname\"]. You may omit the opening and closing bracket.") }} diff --git a/puppetboard/templates/report.html b/puppetboard/templates/report.html index 5e4a185..c640188 100644 --- a/puppetboard/templates/report.html +++ b/puppetboard/templates/report.html @@ -12,7 +12,7 @@ - {{ report.node }} + {{ report.node }} {{report.version}} @@ -48,26 +48,62 @@ {{event.item['old']}} {{event.item['new']}} - {# - -
- {{event.item['message']}} -
- - #} + {% endfor %} + + + +

Logs

+ + + + + + + + + + + + {% for log in logs %} + {% if log.level == 'info' or log.level == 'notice' %} + + {% elif log.level == 'warning' %} + + {% else %} + + {% endif %} + + + + + {% if log.file != None and log.line != None %} + + {% else %} + + {% endif %} + + {% endfor %} + +
TimestampSourceTagsMessageLocation
{{log.time}}{{log.source}}{{log.tags|join(', ')}}{{log.message}}{{log.file}}:{{log.line}}
+ +

Metrics

+ + + + + + + + + + {% for metric in metrics %} + + + + + {% endfor %}
CategoryNameValue
{{metric.category}}{{metric.name}}{{metric.value|round(2)}}
{% endblock content %} -{% block script %} - -{% endblock script %} diff --git a/puppetboard/templates/reports.html b/puppetboard/templates/reports.html index 8109b11..e862380 100644 --- a/puppetboard/templates/reports.html +++ b/puppetboard/templates/reports.html @@ -1,6 +1,6 @@ {% extends 'layout.html' %} +{% import '_macros.html' as macros %} {% block content %} -
- Pending #PDB-201. You can access reports for a node or individual reports through the Nodes tab. -
+{{ macros.reports_table(reports, reports_count, report_event_counts, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True, show_search_bar=True, searchable=True)}} +{{ macros.render_pagination(pagination)}} {% endblock content %} diff --git a/puppetboard/templates/reports_node.html b/puppetboard/templates/reports_node.html deleted file mode 100644 index 885b357..0000000 --- a/puppetboard/templates/reports_node.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends 'layout.html' %} -{% import '_macros.html' as macros %} -{% block content %} -{{ macros.reports_table(reports, nodename, reports_count, condensed=False, hash_truncate=False, show_conf_col=True, show_agent_col=True, show_host_col=True)}} -{% endblock content %} diff --git a/puppetboard/utils.py b/puppetboard/utils.py index 6f30b88..e833b57 100644 --- a/puppetboard/utils.py +++ b/puppetboard/utils.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import json +from math import ceil from requests.exceptions import HTTPError, ConnectionError from pypuppetdb.errors import EmptyResponseError @@ -29,17 +30,6 @@ def get_or_abort(func, *args, **kwargs): abort(204) -def limit_reports(reports, limit): - """Helper to yield a number of from the reports generator. - - This is an ugly solution at best... - """ - for count, report in enumerate(reports): - if count == limit: - raise StopIteration - yield report - - def yield_or_stop(generator): """Similar in intent to get_or_abort this helper will iterate over our generators and handle certain errors. @@ -54,3 +44,35 @@ def yield_or_stop(generator): raise except (EmptyResponseError, ConnectionError, HTTPError): raise StopIteration + +class Pagination(object): + + def __init__(self, page, per_page, total_count): + self.page = page + self.per_page = per_page + self.total_count = total_count + + @property + def pages(self): + return int(ceil(self.total_count / float(self.per_page))) + + @property + def has_prev(self): + return self.page > 1 + + @property + def has_next(self): + return self.page < self.pages + + def iter_pages(self, left_edge=2, left_current=2, + right_current=5, right_edge=2): + last = 0 + for num in xrange(1, self.pages + 1): + if num <= left_edge or \ + (num > self.page - left_current - 1 and \ + num < self.page + right_current) or \ + num > self.pages - right_edge: + if last + 1 != num: + yield None + yield num + last = num diff --git a/setup.py b/setup.py index c7cf0ee..ea9c915 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') sys.exit() -VERSION = "0.0.5" +VERSION = "0.1.0" with codecs.open('README.rst', encoding='utf-8') as f: README = f.read() @@ -23,7 +23,7 @@ setup( author='Daniele Sluijters', author_email='daniele.sluijters+pypi@gmail.com', packages=find_packages(), - url='https://github.com/nedap/puppetboard', + url='https://github.com/puppet-community/puppetboard', license='Apache License 2.0', description='Web frontend for PuppetDB', include_package_data=True, @@ -32,7 +32,7 @@ setup( "Flask >= 0.10.1", "Flask-WTF >= 0.9.4, <= 0.9.5", "WTForms < 2.0", - "pypuppetdb >= 0.1.0, < 0.2.0", + "pypuppetdb >= 0.2.0, < 0.3.0", ], keywords="puppet puppetdb puppetboard", classifiers=[