From f13100664a578f947a9f747c5a94ee92b7400775 Mon Sep 17 00:00:00 2001 From: redref Date: Sun, 5 Feb 2017 13:29:06 +0100 Subject: [PATCH 1/9] Facts page fix + performance revamp Removed facts query to let only fact-names. facts query time grow pretty quickly with number of nodes. Drawback: no filter on environment (which seems acceptable) Add testing about view and column repartition (broken in jinja2 2.9.X / inner loop variables). Rework facts page (jinja 2.9 compliant) --- puppetboard/app.py | 46 ++++++++++++++++---------------- puppetboard/templates/facts.html | 21 +++++---------- test/test_app.py | 19 +++++++++++++ 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index fc45f1c..268320b 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -630,33 +630,33 @@ def facts(env): check_env(env, envs) facts = [] order_by = '[{"field": "name", "order": "asc"}]' + facts = get_or_abort(puppetdb.fact_names) - if env == '*': - facts = get_or_abort(puppetdb.fact_names) - else: - query = ExtractOperator() - query.add_field(str('name')) - query.add_query(EqualsOperator("environment", env)) - query.add_group_by(str("name")) - - for names in get_or_abort(puppetdb._query, - 'facts', - query=query, - order_by=order_by): - facts.append(names['name']) - - facts_dict = collections.defaultdict(list) + facts_columns = [[]] + letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size + count = 0 for fact in facts: - letter = fact[0].upper() - letter_list = facts_dict[letter] - letter_list.append(fact) - facts_dict[letter] = letter_list + count += 1 + + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset + letter = fact[0].upper() + letter_list = [] + + letter_list.append(fact) + facts_columns[-1].append(letter_list) - sorted_facts_dict = sorted(facts_dict.items()) return render_template('facts.html', - facts_dict=sorted_facts_dict, - facts_len=(sum(map(len, facts_dict.values())) + - len(facts_dict) * 5), + facts_columns=facts_columns, envs=envs, current_env=env) diff --git a/puppetboard/templates/facts.html b/puppetboard/templates/facts.html index ce3fe77..7b3a245 100644 --- a/puppetboard/templates/facts.html +++ b/puppetboard/templates/facts.html @@ -4,26 +4,19 @@
+ {%- for column in facts_columns %}
- {%- set facts_count = 0 -%} - {%- set break = facts_len//4 + 1 -%} - {%- for key,facts_list in facts_dict %} + {%- for letter in column %}
- {{key}} + {{ letter[0][0]|upper }}
    - {%- for fact in facts_list %} -
  • {{fact}}
    • + {%- for fact in letter %} +
  • {{ fact }}
    • {%- endfor %}
- {%- set facts_count = facts_count + facts_list|length -%} - {%- if facts_count >= break -%} -
-
- {%- set break = facts_len//4 + 1 + break -%} - {%- endif -%} - {%- set facts_count = facts_count + 5 -%} - {% endfor %} + {%- endfor %}
+ {%- endfor %}
{% endblock content %} diff --git a/test/test_app.py b/test/test_app.py index a3cc494..3b8f77a 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -620,3 +620,22 @@ def test_catalogs_json_compare(client, mocker, "action": "/catalogs/compare/node-unreported...node-%s" % found_status}) assert len(val) == 1 + + +def test_facts_view(client, mocker, mock_puppetdb_environments): + query_data = { + 'fact-names': [[chr(i) for i in range(ord('a'), ord('z') + 1)]] + } + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/facts') + assert rv.status_code == 200 + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + searchable = soup.find('div', {'class': 'searchable'}) + vals = searchable.find_all('div', {'class': 'column'}) + assert len(vals) == 4 From a21bd0ac1d915f3b0bfe41a7625d3f25b5a43d04 Mon Sep 17 00:00:00 2001 From: redref Date: Sun, 5 Feb 2017 18:55:19 +0100 Subject: [PATCH 2/9] Revamp fact pages and tables to datatables --- puppetboard/app.py | 106 ++++++++++++++++++---------- puppetboard/templates/_macros.html | 2 + puppetboard/templates/fact.html | 63 ++++++++--------- puppetboard/templates/fact.json.tpl | 38 ++++++++++ puppetboard/templates/node.html | 17 ++++- 5 files changed, 152 insertions(+), 74 deletions(-) create mode 100644 puppetboard/templates/fact.json.tpl diff --git a/puppetboard/app.py b/puppetboard/app.py index 268320b..615a3a1 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -412,11 +412,10 @@ def node(env, node_name): query.add(EqualsOperator("certname", node_name)) node = get_or_abort(puppetdb.node, node_name) - facts = node.facts() + return render_template( 'node.html', node=node, - facts=yield_or_stop(facts), envs=envs, current_env=env, columns=REPORTS_COLUMNS[:2]) @@ -661,73 +660,102 @@ def facts(env): current_env=env) -@app.route('/fact/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact/') -def fact(env, fact): - """Fetches the specific fact from PuppetDB and displays its value per +@app.route('/fact/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) +@app.route('//fact/', defaults={'value': None}) +@app.route('/fact//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//fact//') +def fact(env, fact, value): + """Fetches the specific fact(/value) from PuppetDB and displays per node for which this fact is known. :param env: Searches for facts in this environment :type env: :obj:`string` :param fact: Find all facts with this name :type fact: :obj:`string` + :param fact: Find all facts with this value + :type fact: :obj:`string` """ envs = environments() check_env(env, envs) - # we can only consume the generator once, lists can be doubly consumed - # om nom nom render_graph = False - if fact in graph_facts: + if fact in graph_facts and not value: render_graph = True - if env == '*': - query = None - else: - query = EqualsOperator("environment", env) - - localfacts = [f for f in yield_or_stop(puppetdb.facts( - name=fact, query=query))] - return Response(stream_with_context(stream_template( + return render_template( 'fact.html', - name=fact, + fact=fact, + value=value, render_graph=render_graph, - facts=localfacts, envs=envs, - current_env=env))) + current_env=env) -@app.route('/fact//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact//') -def fact_value(env, fact, value): - """On asking for fact/value get all nodes with that fact. +@app.route('/fact//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node': None, 'value': None}) +@app.route('//fact//json', defaults={'node': None, 'value': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('//fact///json', defaults={'node': None}) +@app.route('/node//facts/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'fact': None, 'value': None}) +@app.route('//node//facts/json', + defaults={'fact': None, 'value': None}) +def fact_ajax(env, node, fact, value): + """Fetches the specific facts matching (node/fact/value) from PuppetDB and + return a JSON table :param env: Searches for facts in this environment :type env: :obj:`string` + :param fact: Find all facts for this node + :type fact: :obj:`string` :param fact: Find all facts with this name :type fact: :obj:`string` - :param value: Filter facts whose value is equal to this - :type value: :obj:`string` + :param fact: Find all facts with this value + :type fact: :obj:`string` """ + draw = int(request.args.get('draw', 0)) + envs = environments() check_env(env, envs) - if env == '*': - query = None - else: - query = EqualsOperator("environment", env) + render_graph = False + if fact in graph_facts and not value and not node: + render_graph = True - facts = get_or_abort(puppetdb.facts, - name=fact, - value=value, - query=query) - localfacts = [f for f in yield_or_stop(facts)] - return render_template( - 'fact.html', + query = AndOperator() + if node: + query.add(EqualsOperator("certname", node)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + if len(query.operations) == 0: + query = None + + # Generator needs to be converted (graph / total) + facts = [f for f in get_or_abort( + puppetdb.facts, name=fact, value=value, - facts=localfacts, + query=query)] + + total = len(facts) + + return render_template( + 'fact.json.tpl', + fact=fact, + node=node, + value=value, + draw=draw, + total=total, + total_filtered=total, + render_graph=render_graph, + facts=facts, envs=envs, current_env=env) diff --git a/puppetboard/templates/_macros.html b/puppetboard/templates/_macros.html index fe28cc5..e3b31b1 100644 --- a/puppetboard/templates/_macros.html +++ b/puppetboard/templates/_macros.html @@ -76,6 +76,8 @@ // Paging options "lengthMenu": {{ length_selector }}, "pageLength": {{ default_length }}, + // Search as regex (does not apply if serverSide) + "search": {"regex": true}, // Default sort "order": [[ 0, "desc" ]], // Custom options diff --git a/puppetboard/templates/fact.html b/puppetboard/templates/fact.html index ffaa5ce..79bd2c5 100644 --- a/puppetboard/templates/fact.html +++ b/puppetboard/templates/fact.html @@ -1,50 +1,45 @@ {% extends 'layout.html' %} {% import '_macros.html' as macros %} -{% block javascript %} -{% if render_graph %} -var chart = null; -var data = [ -{% for fact in facts|groupby('value') %} - { - label: '{{ fact.grouper.replace("\n", " ") }}', - value: {{ fact.list|length }} - }, -{% endfor %} - { - value: 0, -} -] -var fact_values = data.map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];}); -var realdata = fact_values.slice(0, 15); -var otherdata = fact_values.slice(15); -if (otherdata.length > 0) { - realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]); -} -{% endif %} -{% endblock javascript %} - {% block onload_script %} - $('table').tablesort(); - {% if render_graph %} - chart = c3.generate({ +{% macro extra_options(caller) %} + // No per page AJAX + 'serverSide': false, +{% endmacro %} +{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, fact=fact, value=value), default_length=config.NORMAL_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }} + +{% if render_graph %} +table.on('xhr', function(e, settings, json){ + var fact_values = json['chart'].map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];}); + var realdata = fact_values.slice(0, 15); + var otherdata = fact_values.slice(15); + if (otherdata.length > 0) { + realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]); + } + c3.generate({ bindto: '#factChart', data: { columns: realdata, type : '{{config.GRAPH_TYPE|default('pie')}}', } }); - {% endif %} +}) +{% endif %} {% endblock onload_script %} {% block content %} +{% if render_graph %}
-

{{name}}{% if value %}/{{value}}{% endif %} ({{facts|length}})

- - -{% if value %} -{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, show_value=False, margin_bottom=10)}} -{% else %} -{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, link_facts=True, margin_bottom=10)}} {% endif %} +

{{ fact }}{% if value %}/{{ value }}{% endif %}

+ + + + + {% if not value %}{% endif %} + + + + +
NodeValue
{% endblock content %} diff --git a/puppetboard/templates/fact.json.tpl b/puppetboard/templates/fact.json.tpl new file mode 100644 index 0000000..9e38570 --- /dev/null +++ b/puppetboard/templates/fact.json.tpl @@ -0,0 +1,38 @@ +{ + "draw": {{draw}}, + "recordsTotal": {{total}}, + "recordsFiltered": {{total_filtered}}, + "data": [ + {% for fact_h in facts -%} + {%- if not loop.first %},{%- endif -%} + [ + {%- if not fact -%} + {{ fact_h.name | jsonprint }} + {%- if node or value %},{% endif -%} + {%- endif -%} + {%- if not node -%} + {% filter jsonprint %}{{ fact_h.node }}{% endfilter %} + {%- if not value %},{% endif -%} + {%- endif -%} + {%- if not value -%} + {%- if fact_h.value is mapping -%} + {% filter jsonprint %}
{{ fact_h.value | jsonprint }}
{% endfilter %} + {%- else -%} + {% filter jsonprint %}
{{ fact_h.value }}
{% endfilter %} + {%- endif -%} + {%- endif -%} + ] + {% endfor -%} + ] + {%- if render_graph %}, + "chart": [ + {% for fact_h in facts | groupby('value') -%} + {%- if not loop.first %},{%- endif -%} + { + "label": {{ fact_h.grouper.replace("\n", " ") | jsonprint }}, + "value": {{ fact_h.list|length }} + } + {% endfor %} + ] + {% endif %} +} diff --git a/puppetboard/templates/node.html b/puppetboard/templates/node.html index 507e877..5c51f9b 100644 --- a/puppetboard/templates/node.html +++ b/puppetboard/templates/node.html @@ -17,7 +17,13 @@ 'pagingType': 'simple', "bFilter": false, {% endmacro %} +{% macro facts_extra_options(caller) %} + 'paging': false, + // No per page AJAX + 'serverSide': false, +{% endmacro %} {{ macros.datatable_init(table_html_id="reports_table", ajax_url=url_for('reports_ajax', env=current_env, node_name=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }} +{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, node=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=facts_extra_options) }} {% endblock onload_script %} {% block content %} @@ -69,7 +75,16 @@

Facts

- {{macros.facts_table(facts, link_facts=True, condensed=True, current_env=current_env)}} + + + + + + + + + +
NameValue
{% endblock content %} From 40511c007a590e735065dbc455699610e5151bec Mon Sep 17 00:00:00 2001 From: redref Date: Sun, 5 Feb 2017 20:58:57 +0100 Subject: [PATCH 3/9] Fact pages and node page tests --- puppetboard/app.py | 10 +- puppetboard/templates/_macros.html | 48 -------- test/test_app.py | 186 ++++++++++++++++++++++++++++- 3 files changed, 189 insertions(+), 55 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 615a3a1..147432a 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -391,9 +391,9 @@ def inventory(env): ))) -@app.route('/node//', +@app.route('/node/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//node//') +@app.route('//node/') def node(env, node_name): """Display a dashboard for a node showing as much data as we have on that node. This includes facts and reports but not Resources as that is too @@ -421,11 +421,11 @@ def node(env, node_name): columns=REPORTS_COLUMNS[:2]) -@app.route('/reports/', +@app.route('/reports', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node_name': None}) -@app.route('//reports/', defaults={'node_name': None}) -@app.route('/reports//', +@app.route('//reports', defaults={'node_name': None}) +@app.route('/reports/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//reports/') def reports(env, node_name): diff --git a/puppetboard/templates/_macros.html b/puppetboard/templates/_macros.html index e3b31b1..0e4828d 100644 --- a/puppetboard/templates/_macros.html +++ b/puppetboard/templates/_macros.html @@ -1,51 +1,3 @@ -{% macro facts_table(facts, current_env, autofocus=False, condensed=False, show_node=False, show_value=True, link_facts=False, margin_top=20, margin_bottom=20) -%} -
- -
- - - - {% if show_node %} - - {% else %} - - {% endif %} - {% if show_value %} - - {% endif %} - - - - {% for fact in facts %} - - {% if show_node %} - - {% else %} - - {% endif %} - {% if show_value %} - - {% endif %} - - {% endfor %} - -
NodeFactValue
{{fact.node}}{{fact.name}} - {% if link_facts %} - {% if fact.value is mapping %} - 
{{fact.value|jsonprint}}
 - {% else %} - {{fact.value}} - {% endif %} - {% else %} - {% if fact.value is mapping %} - 
{{fact.value|jsonprint}}
- {% else %} - {{fact.value}} - {% endif %} - {% endif %} -
-{%- endmacro %} - {% macro status_counts(caller, status, node_name, events, current_env, unreported_time=False, report_hash=False) -%} {{ status|upper }} {% if status == 'unreported' %} diff --git a/test/test_app.py b/test/test_app.py index 3b8f77a..357e5ba 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -582,7 +582,7 @@ def test_catalogs_json(client, mocker, found_status = None for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']: val = BeautifulSoup(line[0], 'html.parser').find_all( - 'a', {"href": "/node/node-%s/" % status}) + 'a', {"href": "/node/node-%s" % status}) if len(val) == 1: found_status = status break @@ -609,7 +609,7 @@ def test_catalogs_json_compare(client, mocker, found_status = None for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']: val = BeautifulSoup(line[0], 'html.parser').find_all( - 'a', {"href": "/node/node-%s/" % status}) + 'a', {"href": "/node/node-%s" % status}) if len(val) == 1: found_status = status break @@ -639,3 +639,185 @@ def test_facts_view(client, mocker, mock_puppetdb_environments): searchable = soup.find('div', {'class': 'searchable'}) vals = searchable.find_all('div', {'class': 'column'}) assert len(vals) == 4 + + +def test_fact_view_with_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/fact/architecture') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 1 + + +def test_fact_view_without_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/%2A/fact/augeas') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 0 + + +def test_fact_value_view(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/fact/architecture/amd64') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 0 + + +def test_node_view(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/node/node-failed') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('table', {"id": "facts_table"}) + assert len(vals) == 1 + + vals = soup.find_all('table', {"id": "reports_table"}) + assert len(vals) == 1 + + +def test_fact_json_with_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/fact/architecture/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' in result_json + assert len(result_json['chart']) == 3 + # Test group_by + assert result_json['chart'][1]['value'] == 2 + + +def test_fact_json_without_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/%2A/fact/augeas/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' not in result_json + + +def test_fact_value_json(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/fact/architecture/amd64/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 1 + + assert 'chart' not in result_json + + +def test_node_facts_json(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-failed', + 'name': 'fact-%s' % i, + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/node/node-failed/facts/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' not in result_json From 484727b62c7891c42fede4ecbdc2cb70aa796519 Mon Sep 17 00:00:00 2001 From: redref Date: Sun, 5 Feb 2017 21:27:50 +0100 Subject: [PATCH 4/9] Fix #356 with the new template Created a custom template_filter as in python3, the groupby filter cannot order Bool vs Str. Needed to push format before the groupby which is not currently possible in jinja. --- puppetboard/app.py | 6 ++++++ puppetboard/templates/fact.json.tpl | 4 ++-- test/test_app.py | 8 ++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 147432a..b3354c5 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -78,6 +78,12 @@ def version(): return __version__ +@app.template_filter() +def format_attribute(obj, attr, format_str): + setattr(obj, attr, format_str.format(getattr(obj, attr))) + return obj + + def stream_template(template_name, **context): app.update_template_context(context) t = app.jinja_env.get_template(template_name) diff --git a/puppetboard/templates/fact.json.tpl b/puppetboard/templates/fact.json.tpl index 9e38570..7bd982d 100644 --- a/puppetboard/templates/fact.json.tpl +++ b/puppetboard/templates/fact.json.tpl @@ -26,10 +26,10 @@ ] {%- if render_graph %}, "chart": [ - {% for fact_h in facts | groupby('value') -%} + {% for fact_h in facts | map('format_attribute', 'value', '{0}') | groupby('value') -%} {%- if not loop.first %},{%- endif -%} { - "label": {{ fact_h.grouper.replace("\n", " ") | jsonprint }}, + "label": {{ fact_h.grouper | replace("\n", " ") | jsonprint }}, "value": {{ fact_h.list|length }} } {% endfor %} diff --git a/test/test_app.py b/test/test_app.py index 357e5ba..79e4d1b 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -699,7 +699,7 @@ def test_node_view(client, mocker, def test_fact_json_with_graph(client, mocker, mock_puppetdb_environments, mock_puppetdb_default_nodes): - values = ['a', 'b', 'b', 'd'] + values = ['a', 'b', 'b', 'd', True, 'a\nb'] query_data = {'facts': []} query_data['facts'].append([]) for i, value in enumerate(values): @@ -720,14 +720,14 @@ def test_fact_json_with_graph(client, mocker, result_json = json.loads(rv.data.decode('utf-8')) assert 'data' in result_json - assert len(result_json['data']) == 4 + assert len(result_json['data']) == 6 for line in result_json['data']: assert len(line) == 2 assert 'chart' in result_json - assert len(result_json['chart']) == 3 + assert len(result_json['chart']) == 5 # Test group_by - assert result_json['chart'][1]['value'] == 2 + assert result_json['chart'][3]['value'] == 2 def test_fact_json_without_graph(client, mocker, From 4b96cfe196d94ca73abb2338535154c599b648b7 Mon Sep 17 00:00:00 2001 From: redref Date: Thu, 23 Mar 2017 22:05:59 +0100 Subject: [PATCH 5/9] Fix facts method doc --- puppetboard/app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index b3354c5..ee369e4 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -680,8 +680,8 @@ def fact(env, fact, value): :type env: :obj:`string` :param fact: Find all facts with this name :type fact: :obj:`string` - :param fact: Find all facts with this value - :type fact: :obj:`string` + :param value: Find all facts with this value + :type value: :obj:`string` """ envs = environments() check_env(env, envs) @@ -717,12 +717,12 @@ def fact_ajax(env, node, fact, value): :param env: Searches for facts in this environment :type env: :obj:`string` - :param fact: Find all facts for this node - :type fact: :obj:`string` + :param node: Find all facts for this node + :type node: :obj:`string` :param fact: Find all facts with this name :type fact: :obj:`string` - :param fact: Find all facts with this value - :type fact: :obj:`string` + :param value: Filter facts whose value is equal to this + :type value: :obj:`string` """ draw = int(request.args.get('draw', 0)) From d4d7b6a56a987f333b5ee843e6d3f80823e1d12e Mon Sep 17 00:00:00 2001 From: redref Date: Thu, 23 Mar 2017 22:55:30 +0100 Subject: [PATCH 6/9] Clean CatalogForm --- puppetboard/app.py | 2 +- puppetboard/forms.py | 6 ------ test/test_form.py | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index ee369e4..22e67bf 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -19,7 +19,7 @@ from flask import ( from pypuppetdb import connect from pypuppetdb.QueryBuilder import * -from puppetboard.forms import (CatalogForm, QueryForm) +from puppetboard.forms import QueryForm from puppetboard.utils import ( get_or_abort, yield_or_stop, get_db_version, jsonprint, prettyprint diff --git a/puppetboard/forms.py b/puppetboard/forms.py index 72f0aba..b67d441 100644 --- a/puppetboard/forms.py +++ b/puppetboard/forms.py @@ -28,9 +28,3 @@ class QueryForm(FlaskForm): ('pql', 'PQL'), ]) rawjson = BooleanField('Raw JSON') - - -class CatalogForm(FlaskForm): - """The form used to compare the catalogs of different nodes.""" - compare = HiddenField('compare') - against = SelectField('against') diff --git a/test/test_form.py b/test/test_form.py index 35286f5..10b8a3c 100644 --- a/test/test_form.py +++ b/test/test_form.py @@ -3,7 +3,7 @@ from puppetboard import app, forms def test_form_valid(capsys): - for form in [forms.QueryForm, forms.CatalogForm]: + for form in [forms.QueryForm]: with app.app.test_request_context(): qf = form() out, err = capsys.readouterr() From b17a2b0450cddc61533b12e0603084bb14588b51 Mon Sep 17 00:00:00 2001 From: redref Date: Thu, 23 Mar 2017 23:08:13 +0100 Subject: [PATCH 7/9] Fix tox deps --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 1bbe8bf..36762a2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,9 @@ envlist = py{26,27,35,36} [testenv] +deps= + -rrequirements-test.txt + bandit commands= py.test --cov=puppetboard --pep8 -v py{27,35,36}: bandit -r puppetboard From c1fd33fb5c63eedf0dce9f4d09784782d7485ef2 Mon Sep 17 00:00:00 2001 From: redref Date: Thu, 23 Mar 2017 23:08:37 +0100 Subject: [PATCH 8/9] Generate fact JSON directly in python --- puppetboard/app.py | 50 ++++++++++++++++++----------- puppetboard/templates/fact.json.tpl | 38 ---------------------- test/test_app.py | 2 +- 3 files changed, 33 insertions(+), 57 deletions(-) delete mode 100644 puppetboard/templates/fact.json.tpl diff --git a/puppetboard/app.py b/puppetboard/app.py index 22e67bf..9c6bf02 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -78,12 +78,6 @@ def version(): return __version__ -@app.template_filter() -def format_attribute(obj, attr, format_str): - setattr(obj, attr, format_str.format(getattr(obj, attr))) - return obj - - def stream_template(template_name, **context): app.update_template_context(context) t = app.jinja_env.get_template(template_name) @@ -752,18 +746,38 @@ def fact_ajax(env, node, fact, value): total = len(facts) - return render_template( - 'fact.json.tpl', - fact=fact, - node=node, - value=value, - draw=draw, - total=total, - total_filtered=total, - render_graph=render_graph, - facts=facts, - envs=envs, - current_env=env) + counts = {} + json = { + 'draw': draw, + 'recordsTotal': total, + 'recordsFiltered': total, + 'data': []} + + for fact_h in facts: + line = [] + if not fact: + line.append(fact_h.name) + if not node: + line.append(''.format(url_for( + 'node', env=env, node_name=fact_h.node))) + if not value: + line.append(''.format(url_for( + 'fact', env=env, fact=fact_h.name, value=fact_h.value))) + + json['data'].append(line) + + if render_graph: + if fact_h.value not in counts: + counts[fact_h.value] = 0 + counts[fact_h.value] += 1 + + if render_graph: + json['chart'] = [ + {"label": "{0}".format(k).replace('\n', ' '), + "value": counts[k]} + for k in sorted(counts, key=lambda k: counts[k], reverse=True)] + + return jsonify(json) @app.route('/query', methods=('GET', 'POST'), diff --git a/puppetboard/templates/fact.json.tpl b/puppetboard/templates/fact.json.tpl deleted file mode 100644 index 7bd982d..0000000 --- a/puppetboard/templates/fact.json.tpl +++ /dev/null @@ -1,38 +0,0 @@ -{ - "draw": {{draw}}, - "recordsTotal": {{total}}, - "recordsFiltered": {{total_filtered}}, - "data": [ - {% for fact_h in facts -%} - {%- if not loop.first %},{%- endif -%} - [ - {%- if not fact -%} - {{ fact_h.name | jsonprint }} - {%- if node or value %},{% endif -%} - {%- endif -%} - {%- if not node -%} - {% filter jsonprint %}{{ fact_h.node }}{% endfilter %} - {%- if not value %},{% endif -%} - {%- endif -%} - {%- if not value -%} - {%- if fact_h.value is mapping -%} - {% filter jsonprint %}
{{ fact_h.value | jsonprint }}
{% endfilter %} - {%- else -%} - {% filter jsonprint %}
{{ fact_h.value }}
{% endfilter %} - {%- endif -%} - {%- endif -%} - ] - {% endfor -%} - ] - {%- if render_graph %}, - "chart": [ - {% for fact_h in facts | map('format_attribute', 'value', '{0}') | groupby('value') -%} - {%- if not loop.first %},{%- endif -%} - { - "label": {{ fact_h.grouper | replace("\n", " ") | jsonprint }}, - "value": {{ fact_h.list|length }} - } - {% endfor %} - ] - {% endif %} -} diff --git a/test/test_app.py b/test/test_app.py index 79e4d1b..7a6acd6 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -727,7 +727,7 @@ def test_fact_json_with_graph(client, mocker, assert 'chart' in result_json assert len(result_json['chart']) == 5 # Test group_by - assert result_json['chart'][3]['value'] == 2 + assert result_json['chart'][0]['value'] == 2 def test_fact_json_without_graph(client, mocker, From d2c47df31f895253d8982a1f3e748766ac25c613 Mon Sep 17 00:00:00 2001 From: redref Date: Mon, 3 Apr 2017 22:37:11 +0200 Subject: [PATCH 9/9] Fix link formatting --- puppetboard/app.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/puppetboard/app.py b/puppetboard/app.py index 9c6bf02..f1c0363 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -758,11 +758,14 @@ def fact_ajax(env, node, fact, value): if not fact: line.append(fact_h.name) if not node: - line.append(''.format(url_for( - 'node', env=env, node_name=fact_h.node))) + line.append('{1}'.format( + url_for('node', env=env, node_name=fact_h.node), + fact_h.node)) if not value: - line.append(''.format(url_for( - 'fact', env=env, fact=fact_h.name, value=fact_h.value))) + line.append('{1}'.format( + url_for( + 'fact', env=env, fact=fact_h.name, value=fact_h.value), + fact_h.value)) json['data'].append(line)