diff --git a/puppetboard/app.py b/puppetboard/app.py index 03db1bb..52dc51e 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -19,7 +19,7 @@ from flask import ( from pypuppetdb import connect from pypuppetdb.QueryBuilder import * -from puppetboard.forms import (CatalogForm, QueryForm) +from puppetboard.forms import QueryForm from puppetboard.utils import ( get_or_abort, yield_or_stop, get_db_version, jsonprint, prettyprint @@ -406,9 +406,9 @@ def inventory_ajax(env): columns=fact_names) -@app.route('/node//', +@app.route('/node/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//node//') +@app.route('//node/') def node(env, node_name): """Display a dashboard for a node showing as much data as we have on that node. This includes facts and reports but not Resources as that is too @@ -427,21 +427,20 @@ def node(env, node_name): query.add(EqualsOperator("certname", node_name)) node = get_or_abort(puppetdb.node, node_name) - facts = node.facts() + return render_template( 'node.html', node=node, - facts=yield_or_stop(facts), envs=envs, current_env=env, columns=REPORTS_COLUMNS[:2]) -@app.route('/reports/', +@app.route('/reports', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node_name': None}) -@app.route('//reports/', defaults={'node_name': None}) -@app.route('/reports//', +@app.route('//reports', defaults={'node_name': None}) +@app.route('/reports/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//reports/') def reports(env, node_name): @@ -638,108 +637,160 @@ def facts(env): check_env(env, envs) facts = [] order_by = '[{"field": "name", "order": "asc"}]' + facts = get_or_abort(puppetdb.fact_names) - if env == '*': - facts = get_or_abort(puppetdb.fact_names) - else: - query = ExtractOperator() - query.add_field(str('name')) - query.add_query(EqualsOperator("environment", env)) - query.add_group_by(str("name")) - - for names in get_or_abort(puppetdb._query, - 'facts', - query=query, - order_by=order_by): - facts.append(names['name']) - - facts_dict = collections.defaultdict(list) + facts_columns = [[]] + letter = None + letter_list = None + break_size = (len(facts) / 4) + 1 + next_break = break_size + count = 0 for fact in facts: - letter = fact[0].upper() - letter_list = facts_dict[letter] - letter_list.append(fact) - facts_dict[letter] = letter_list + count += 1 + + if letter != fact[0].upper() or not letter: + if count > next_break: + # Create a new column + facts_columns.append([]) + next_break += break_size + if letter_list: + facts_columns[-1].append(letter_list) + # Reset + letter = fact[0].upper() + letter_list = [] + + letter_list.append(fact) + facts_columns[-1].append(letter_list) - sorted_facts_dict = sorted(facts_dict.items()) return render_template('facts.html', - facts_dict=sorted_facts_dict, - facts_len=(sum(map(len, facts_dict.values())) + - len(facts_dict) * 5), + facts_columns=facts_columns, envs=envs, current_env=env) -@app.route('/fact/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact/') -def fact(env, fact): - """Fetches the specific fact from PuppetDB and displays its value per +@app.route('/fact/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None}) +@app.route('//fact/', defaults={'value': None}) +@app.route('/fact//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('//fact//') +def fact(env, fact, value): + """Fetches the specific fact(/value) from PuppetDB and displays per node for which this fact is known. :param env: Searches for facts in this environment :type env: :obj:`string` :param fact: Find all facts with this name :type fact: :obj:`string` - """ - envs = environments() - check_env(env, envs) - - # we can only consume the generator once, lists can be doubly consumed - # om nom nom - render_graph = False - if fact in graph_facts: - render_graph = True - - if env == '*': - query = None - else: - query = EqualsOperator("environment", env) - - localfacts = [f for f in yield_or_stop(puppetdb.facts( - name=fact, query=query))] - return Response(stream_with_context(stream_template( - 'fact.html', - name=fact, - render_graph=render_graph, - facts=localfacts, - envs=envs, - current_env=env))) - - -@app.route('/fact//', - defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) -@app.route('//fact//') -def fact_value(env, fact, value): - """On asking for fact/value get all nodes with that fact. - - :param env: Searches for facts in this environment - :type env: :obj:`string` - :param fact: Find all facts with this name - :type fact: :obj:`string` - :param value: Filter facts whose value is equal to this + :param value: Find all facts with this value :type value: :obj:`string` """ envs = environments() check_env(env, envs) - if env == '*': - query = None - else: - query = EqualsOperator("environment", env) + render_graph = False + if fact in graph_facts and not value: + render_graph = True - facts = get_or_abort(puppetdb.facts, - name=fact, - value=value, - query=query) - localfacts = [f for f in yield_or_stop(facts)] return render_template( 'fact.html', - name=fact, + fact=fact, value=value, - facts=localfacts, + render_graph=render_graph, envs=envs, current_env=env) +@app.route('/fact//json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'node': None, 'value': None}) +@app.route('//fact//json', defaults={'node': None, 'value': None}) +@app.route('/fact///json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None}) +@app.route('//fact///json', defaults={'node': None}) +@app.route('/node//facts/json', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], + 'fact': None, 'value': None}) +@app.route('//node//facts/json', + defaults={'fact': None, 'value': None}) +def fact_ajax(env, node, fact, value): + """Fetches the specific facts matching (node/fact/value) from PuppetDB and + return a JSON table + + :param env: Searches for facts in this environment + :type env: :obj:`string` + :param node: Find all facts for this node + :type node: :obj:`string` + :param fact: Find all facts with this name + :type fact: :obj:`string` + :param value: Filter facts whose value is equal to this + :type value: :obj:`string` + """ + draw = int(request.args.get('draw', 0)) + + envs = environments() + check_env(env, envs) + + render_graph = False + if fact in graph_facts and not value and not node: + render_graph = True + + query = AndOperator() + if node: + query.add(EqualsOperator("certname", node)) + + if env != '*': + query.add(EqualsOperator("environment", env)) + + if len(query.operations) == 0: + query = None + + # Generator needs to be converted (graph / total) + facts = [f for f in get_or_abort( + puppetdb.facts, + name=fact, + value=value, + query=query)] + + total = len(facts) + + counts = {} + json = { + 'draw': draw, + 'recordsTotal': total, + 'recordsFiltered': total, + 'data': []} + + for fact_h in facts: + line = [] + if not fact: + line.append(fact_h.name) + if not node: + line.append('{1}'.format( + url_for('node', env=env, node_name=fact_h.node), + fact_h.node)) + if not value: + line.append('{1}'.format( + url_for( + 'fact', env=env, fact=fact_h.name, value=fact_h.value), + fact_h.value)) + + json['data'].append(line) + + if render_graph: + if fact_h.value not in counts: + counts[fact_h.value] = 0 + counts[fact_h.value] += 1 + + if render_graph: + json['chart'] = [ + {"label": "{0}".format(k).replace('\n', ' '), + "value": counts[k]} + for k in sorted(counts, key=lambda k: counts[k], reverse=True)] + + return jsonify(json) + + @app.route('/query', methods=('GET', 'POST'), defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//query', methods=('GET', 'POST')) diff --git a/puppetboard/forms.py b/puppetboard/forms.py index 72f0aba..b67d441 100644 --- a/puppetboard/forms.py +++ b/puppetboard/forms.py @@ -28,9 +28,3 @@ class QueryForm(FlaskForm): ('pql', 'PQL'), ]) rawjson = BooleanField('Raw JSON') - - -class CatalogForm(FlaskForm): - """The form used to compare the catalogs of different nodes.""" - compare = HiddenField('compare') - against = SelectField('against') diff --git a/puppetboard/templates/_macros.html b/puppetboard/templates/_macros.html index e4cb163..65defad 100644 --- a/puppetboard/templates/_macros.html +++ b/puppetboard/templates/_macros.html @@ -1,51 +1,3 @@ -{% macro facts_table(facts, current_env, autofocus=False, condensed=False, show_node=False, show_value=True, link_facts=False, margin_top=20, margin_bottom=20) -%} -
- -
- - - - {% if show_node %} - - {% else %} - - {% endif %} - {% if show_value %} - - {% endif %} - - - - {% for fact in facts %} - - {% if show_node %} - - {% else %} - - {% endif %} - {% if show_value %} - - {% endif %} - - {% endfor %} - -
NodeFactValue
{{fact.node}}{{fact.name}} - {% if link_facts %} - {% if fact.value is mapping %} - 
{{fact.value|jsonprint}}
 - {% else %} - {{fact.value}} - {% endif %} - {% else %} - {% if fact.value is mapping %} - 
{{fact.value|jsonprint}}
- {% else %} - {{fact.value}} - {% endif %} - {% endif %} -
-{%- endmacro %} - {% macro status_counts(caller, status, node_name, events, current_env, unreported_time=False, report_hash=False) -%} {{ status|upper }} {% if status == 'unreported' %} @@ -99,6 +51,8 @@ // Paging options "lengthMenu": {{ length_selector }}, "pageLength": {{ default_length }}, + // Search as regex (does not apply if serverSide) + "search": {"regex": true}, // Default sort "order": [[ 0, "desc" ]], // Custom options diff --git a/puppetboard/templates/fact.html b/puppetboard/templates/fact.html index ffaa5ce..79bd2c5 100644 --- a/puppetboard/templates/fact.html +++ b/puppetboard/templates/fact.html @@ -1,50 +1,45 @@ {% extends 'layout.html' %} {% import '_macros.html' as macros %} -{% block javascript %} -{% if render_graph %} -var chart = null; -var data = [ -{% for fact in facts|groupby('value') %} - { - label: '{{ fact.grouper.replace("\n", " ") }}', - value: {{ fact.list|length }} - }, -{% endfor %} - { - value: 0, -} -] -var fact_values = data.map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];}); -var realdata = fact_values.slice(0, 15); -var otherdata = fact_values.slice(15); -if (otherdata.length > 0) { - realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]); -} -{% endif %} -{% endblock javascript %} - {% block onload_script %} - $('table').tablesort(); - {% if render_graph %} - chart = c3.generate({ +{% macro extra_options(caller) %} + // No per page AJAX + 'serverSide': false, +{% endmacro %} +{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, fact=fact, value=value), default_length=config.NORMAL_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }} + +{% if render_graph %} +table.on('xhr', function(e, settings, json){ + var fact_values = json['chart'].map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];}); + var realdata = fact_values.slice(0, 15); + var otherdata = fact_values.slice(15); + if (otherdata.length > 0) { + realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]); + } + c3.generate({ bindto: '#factChart', data: { columns: realdata, type : '{{config.GRAPH_TYPE|default('pie')}}', } }); - {% endif %} +}) +{% endif %} {% endblock onload_script %} {% block content %} +{% if render_graph %}
-

{{name}}{% if value %}/{{value}}{% endif %} ({{facts|length}})

- - -{% if value %} -{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, show_value=False, margin_bottom=10)}} -{% else %} -{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, link_facts=True, margin_bottom=10)}} {% endif %} +

{{ fact }}{% if value %}/{{ value }}{% endif %}

+ + + + + {% if not value %}{% endif %} + + + + +
NodeValue
{% endblock content %} diff --git a/puppetboard/templates/facts.html b/puppetboard/templates/facts.html index ce3fe77..7b3a245 100644 --- a/puppetboard/templates/facts.html +++ b/puppetboard/templates/facts.html @@ -4,26 +4,19 @@
+ {%- for column in facts_columns %}
- {%- set facts_count = 0 -%} - {%- set break = facts_len//4 + 1 -%} - {%- for key,facts_list in facts_dict %} + {%- for letter in column %}
- {{key}} + {{ letter[0][0]|upper }}
    - {%- for fact in facts_list %} -
  • {{fact}}
    • + {%- for fact in letter %} +
  • {{ fact }}
    • {%- endfor %}
- {%- set facts_count = facts_count + facts_list|length -%} - {%- if facts_count >= break -%} -
-
- {%- set break = facts_len//4 + 1 + break -%} - {%- endif -%} - {%- set facts_count = facts_count + 5 -%} - {% endfor %} + {%- endfor %}
+ {%- endfor %}
{% endblock content %} diff --git a/puppetboard/templates/node.html b/puppetboard/templates/node.html index 507e877..5c51f9b 100644 --- a/puppetboard/templates/node.html +++ b/puppetboard/templates/node.html @@ -17,7 +17,13 @@ 'pagingType': 'simple', "bFilter": false, {% endmacro %} +{% macro facts_extra_options(caller) %} + 'paging': false, + // No per page AJAX + 'serverSide': false, +{% endmacro %} {{ macros.datatable_init(table_html_id="reports_table", ajax_url=url_for('reports_ajax', env=current_env, node_name=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }} +{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, node=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=facts_extra_options) }} {% endblock onload_script %} {% block content %} @@ -69,7 +75,16 @@

Facts

- {{macros.facts_table(facts, link_facts=True, condensed=True, current_env=current_env)}} + + + + + + + + + +
NameValue
{% endblock content %} diff --git a/test/test_app.py b/test/test_app.py index fc1ec6e..beaa5e9 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -582,7 +582,7 @@ def test_catalogs_json(client, mocker, found_status = None for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']: val = BeautifulSoup(line[0], 'html.parser').find_all( - 'a', {"href": "/node/node-%s/" % status}) + 'a', {"href": "/node/node-%s" % status}) if len(val) == 1: found_status = status break @@ -609,7 +609,7 @@ def test_catalogs_json_compare(client, mocker, found_status = None for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']: val = BeautifulSoup(line[0], 'html.parser').find_all( - 'a', {"href": "/node/node-%s/" % status}) + 'a', {"href": "/node/node-%s" % status}) if len(val) == 1: found_status = status break @@ -622,47 +622,202 @@ def test_catalogs_json_compare(client, mocker, assert len(val) == 1 -def test_inventory_view(client, mocker, mock_puppetdb_environments): - rv = client.get('/inventory') - assert rv.status_code == 200 - soup = BeautifulSoup(rv.data, 'html.parser') - assert soup.title.contents[0] == 'Puppetboard' - - -def test_inventory_json(client, mocker, mock_puppetdb_environments): - facts = ['fqdn', 'ipaddress', 'lsbdistdescription', 'hardwaremodel', - 'kernelrelease', 'puppetversion'] - values = [ - ['node-1', 'X.X.X.X', 'os7', 'server', '4.3', 'X.X.X'], - ['node-2', 'X.X.X.X', 'os5', 'server', '4.1', 'X.X.X'], - ['node-3', 'X.X.X.X', 'os6', 'server', '4.2', 'X.X.X'], - ['node-4', 'X.X.X.X', 'os4', 'server', '4.3', 'X.X.X'], - ] - query_data = {'facts': []} - query_data['facts'].append([]) - - for i, value in enumerate(values): - for idx, column in enumerate(facts): - query_data['facts'][0].append({ - 'certname': value[0], - 'name': column, - 'value': value[idx], - 'environment': 'production' - }) +def test_facts_view(client, mocker, mock_puppetdb_environments): + query_data = { + 'fact-names': [[chr(i) for i in range(ord('a'), ord('z') + 1)]] + } dbquery = MockDbQuery(query_data) mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) - rv = client.get('/inventory/json') + rv = client.get('/facts') assert rv.status_code == 200 - result_json = json.loads(rv.data.decode('utf-8')) - assert 'data' in result_json + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' - for value in values: - for line in result_json['data']: - if value[0] in line[0]: - assert line[1:] == value[1:] - break - else: - raise Exception("Input %s not found" % value) + searchable = soup.find('div', {'class': 'searchable'}) + vals = searchable.find_all('div', {'class': 'column'}) + assert len(vals) == 4 + + +def test_fact_view_with_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/fact/architecture') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 1 + + +def test_fact_view_without_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/%2A/fact/augeas') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 0 + + +def test_fact_value_view(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/fact/architecture/amd64') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('div', {"id": "factChart"}) + assert len(vals) == 0 + + +def test_node_view(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + rv = client.get('/node/node-failed') + assert rv.status_code == 200 + + soup = BeautifulSoup(rv.data, 'html.parser') + assert soup.title.contents[0] == 'Puppetboard' + + vals = soup.find_all('table', {"id": "facts_table"}) + assert len(vals) == 1 + + vals = soup.find_all('table', {"id": "reports_table"}) + assert len(vals) == 1 + + +def test_fact_json_with_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd', True, 'a\nb'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/fact/architecture/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 6 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' in result_json + assert len(result_json['chart']) == 5 + # Test group_by + assert result_json['chart'][0]['value'] == 2 + + +def test_fact_json_without_graph(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/%2A/fact/augeas/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' not in result_json + + +def test_fact_value_json(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-%s' % i, + 'name': 'architecture', + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/fact/architecture/amd64/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 1 + + assert 'chart' not in result_json + + +def test_node_facts_json(client, mocker, + mock_puppetdb_environments, + mock_puppetdb_default_nodes): + values = ['a', 'b', 'b', 'd'] + query_data = {'facts': []} + query_data['facts'].append([]) + for i, value in enumerate(values): + query_data['facts'][0].append({ + 'certname': 'node-failed', + 'name': 'fact-%s' % i, + 'value': value, + 'environment': 'production' + }) + + dbquery = MockDbQuery(query_data) + + mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get) + + rv = client.get('/node/node-failed/facts/json') + assert rv.status_code == 200 + + result_json = json.loads(rv.data.decode('utf-8')) + + assert 'data' in result_json + assert len(result_json['data']) == 4 + for line in result_json['data']: + assert len(line) == 2 + + assert 'chart' not in result_json \ No newline at end of file diff --git a/test/test_form.py b/test/test_form.py index 35286f5..10b8a3c 100644 --- a/test/test_form.py +++ b/test/test_form.py @@ -3,7 +3,7 @@ from puppetboard import app, forms def test_form_valid(capsys): - for form in [forms.QueryForm, forms.CatalogForm]: + for form in [forms.QueryForm]: with app.app.test_request_context(): qf = form() out, err = capsys.readouterr() diff --git a/tox.ini b/tox.ini index 1bbe8bf..36762a2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,9 @@ envlist = py{26,27,35,36} [testenv] +deps= + -rrequirements-test.txt + bandit commands= py.test --cov=puppetboard --pep8 -v py{27,35,36}: bandit -r puppetboard