Merge pull request #357 from redref/facts

Facts revamp with datatables.
This commit is contained in:
Mike Terzo
2017-06-10 03:43:15 -04:00
committed by GitHub
9 changed files with 386 additions and 226 deletions

View File

@@ -19,7 +19,7 @@ from flask import (
from pypuppetdb import connect
from pypuppetdb.QueryBuilder import *
from puppetboard.forms import (CatalogForm, QueryForm)
from puppetboard.forms import QueryForm
from puppetboard.utils import (
get_or_abort, yield_or_stop, get_db_version,
jsonprint, prettyprint
@@ -406,9 +406,9 @@ def inventory_ajax(env):
columns=fact_names)
@app.route('/node/<node_name>/',
@app.route('/node/<node_name>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/node/<node_name>/')
@app.route('/<env>/node/<node_name>')
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
@@ -427,21 +427,20 @@ def node(env, node_name):
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
envs=envs,
current_env=env,
columns=REPORTS_COLUMNS[:2])
@app.route('/reports/',
@app.route('/reports',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
'node_name': None})
@app.route('/<env>/reports/', defaults={'node_name': None})
@app.route('/reports/<node_name>/',
@app.route('/<env>/reports', defaults={'node_name': None})
@app.route('/reports/<node_name>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/reports/<node_name>')
def reports(env, node_name):
@@ -638,108 +637,160 @@ def facts(env):
check_env(env, envs)
facts = []
order_by = '[{"field": "name", "order": "asc"}]'
facts = get_or_abort(puppetdb.fact_names)
if env == '*':
facts = get_or_abort(puppetdb.fact_names)
else:
query = ExtractOperator()
query.add_field(str('name'))
query.add_query(EqualsOperator("environment", env))
query.add_group_by(str("name"))
for names in get_or_abort(puppetdb._query,
'facts',
query=query,
order_by=order_by):
facts.append(names['name'])
facts_dict = collections.defaultdict(list)
facts_columns = [[]]
letter = None
letter_list = None
break_size = (len(facts) / 4) + 1
next_break = break_size
count = 0
for fact in facts:
letter = fact[0].upper()
letter_list = facts_dict[letter]
letter_list.append(fact)
facts_dict[letter] = letter_list
count += 1
if letter != fact[0].upper() or not letter:
if count > next_break:
# Create a new column
facts_columns.append([])
next_break += break_size
if letter_list:
facts_columns[-1].append(letter_list)
# Reset
letter = fact[0].upper()
letter_list = []
letter_list.append(fact)
facts_columns[-1].append(letter_list)
sorted_facts_dict = sorted(facts_dict.items())
return render_template('facts.html',
facts_dict=sorted_facts_dict,
facts_len=(sum(map(len, facts_dict.values())) +
len(facts_dict) * 5),
facts_columns=facts_columns,
envs=envs,
current_env=env)
@app.route('/fact/<fact>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/fact/<fact>')
def fact(env, fact):
"""Fetches the specific fact from PuppetDB and displays its value per
@app.route('/fact/<fact>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None})
@app.route('/<env>/fact/<fact>', defaults={'value': None})
@app.route('/fact/<fact>/<value>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/fact/<fact>/<value>')
def fact(env, fact, value):
"""Fetches the specific fact(/value) from PuppetDB and displays per
node for which this fact is known.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
"""
envs = environments()
check_env(env, envs)
# we can only consume the generator once, lists can be doubly consumed
# om nom nom
render_graph = False
if fact in graph_facts:
render_graph = True
if env == '*':
query = None
else:
query = EqualsOperator("environment", env)
localfacts = [f for f in yield_or_stop(puppetdb.facts(
name=fact, query=query))]
return Response(stream_with_context(stream_template(
'fact.html',
name=fact,
render_graph=render_graph,
facts=localfacts,
envs=envs,
current_env=env)))
@app.route('/fact/<fact>/<value>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/fact/<fact>/<value>')
def fact_value(env, fact, value):
"""On asking for fact/value get all nodes with that fact.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
:param value: Filter facts whose value is equal to this
:param value: Find all facts with this value
:type value: :obj:`string`
"""
envs = environments()
check_env(env, envs)
if env == '*':
query = None
else:
query = EqualsOperator("environment", env)
render_graph = False
if fact in graph_facts and not value:
render_graph = True
facts = get_or_abort(puppetdb.facts,
name=fact,
value=value,
query=query)
localfacts = [f for f in yield_or_stop(facts)]
return render_template(
'fact.html',
name=fact,
fact=fact,
value=value,
facts=localfacts,
render_graph=render_graph,
envs=envs,
current_env=env)
@app.route('/fact/<fact>/json',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
'node': None, 'value': None})
@app.route('/<env>/fact/<fact>/json', defaults={'node': None, 'value': None})
@app.route('/fact/<fact>/<value>/json',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None})
@app.route('/<env>/fact/<fact>/<value>/json', defaults={'node': None})
@app.route('/node/<node>/facts/json',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
'fact': None, 'value': None})
@app.route('/<env>/node/<node>/facts/json',
defaults={'fact': None, 'value': None})
def fact_ajax(env, node, fact, value):
"""Fetches the specific facts matching (node/fact/value) from PuppetDB and
return a JSON table
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param node: Find all facts for this node
:type node: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
:param value: Filter facts whose value is equal to this
:type value: :obj:`string`
"""
draw = int(request.args.get('draw', 0))
envs = environments()
check_env(env, envs)
render_graph = False
if fact in graph_facts and not value and not node:
render_graph = True
query = AndOperator()
if node:
query.add(EqualsOperator("certname", node))
if env != '*':
query.add(EqualsOperator("environment", env))
if len(query.operations) == 0:
query = None
# Generator needs to be converted (graph / total)
facts = [f for f in get_or_abort(
puppetdb.facts,
name=fact,
value=value,
query=query)]
total = len(facts)
counts = {}
json = {
'draw': draw,
'recordsTotal': total,
'recordsFiltered': total,
'data': []}
for fact_h in facts:
line = []
if not fact:
line.append(fact_h.name)
if not node:
line.append('<a href="{0}">{1}</a>'.format(
url_for('node', env=env, node_name=fact_h.node),
fact_h.node))
if not value:
line.append('<a href="{0}">{1}</a>'.format(
url_for(
'fact', env=env, fact=fact_h.name, value=fact_h.value),
fact_h.value))
json['data'].append(line)
if render_graph:
if fact_h.value not in counts:
counts[fact_h.value] = 0
counts[fact_h.value] += 1
if render_graph:
json['chart'] = [
{"label": "{0}".format(k).replace('\n', ' '),
"value": counts[k]}
for k in sorted(counts, key=lambda k: counts[k], reverse=True)]
return jsonify(json)
@app.route('/query', methods=('GET', 'POST'),
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/query', methods=('GET', 'POST'))

View File

@@ -28,9 +28,3 @@ class QueryForm(FlaskForm):
('pql', 'PQL'),
])
rawjson = BooleanField('Raw JSON')
class CatalogForm(FlaskForm):
"""The form used to compare the catalogs of different nodes."""
compare = HiddenField('compare')
against = SelectField('against')

View File

@@ -1,51 +1,3 @@
{% macro facts_table(facts, current_env, autofocus=False, condensed=False, show_node=False, show_value=True, link_facts=False, margin_top=20, margin_bottom=20) -%}
<div class="ui fluid icon input hide" style="margin-bottom:20px">
<input {% if autofocus %} autofocus="autofocus" {% endif %} class="filter-table" placeholder="Type here to filter...">
</div>
<table class="ui very basic {% if condensed %}very{% endif%} compact sortable table" style="table-layout: fixed;">
<thead>
<tr>
{% if show_node %}
<th>Node</th>
{% else %}
<th class="default-sort">Fact</th>
{% endif %}
{% if show_value %}
<th>Value</th>
{% endif %}
</tr>
</thead>
<tbody class="searchable">
{% for fact in facts %}
<tr>
{% if show_node %}
<td><a href="{{url_for('node', env=current_env, node_name=fact.node)}}">{{fact.node}}</a></td>
{% else %}
<td><a href="{{url_for('fact', env=current_env, fact=fact.name)}}">{{fact.name}}</a></td>
{% endif %}
{% if show_value %}
<td style="word-wrap:break-word">
{% if link_facts %}
{% if fact.value is mapping %}
<a href="{{url_for('fact_value', env=current_env, fact=fact.name, value=fact.value)}}"><pre>{{fact.value|jsonprint}}</pre></a>
{% else %}
<a href="{{url_for('fact_value', env=current_env, fact=fact.name, value=fact.value)}}">{{fact.value}}</a>
{% endif %}
{% else %}
{% if fact.value is mapping %}
<pre>{{fact.value|jsonprint}}</pre>
{% else %}
{{fact.value}}
{% endif %}
{% endif %}
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{%- endmacro %}
{% macro status_counts(caller, status, node_name, events, current_env, unreported_time=False, report_hash=False) -%}
<a class="ui {{status}} label status" href="{{url_for('report', env=current_env, node_name=node_name, report_id=report_hash)}}">{{ status|upper }}</a>
{% if status == 'unreported' %}
@@ -99,6 +51,8 @@
// Paging options
"lengthMenu": {{ length_selector }},
"pageLength": {{ default_length }},
// Search as regex (does not apply if serverSide)
"search": {"regex": true},
// Default sort
"order": [[ 0, "desc" ]],
// Custom options

View File

@@ -1,50 +1,45 @@
{% extends 'layout.html' %}
{% import '_macros.html' as macros %}
{% block javascript %}
{% if render_graph %}
var chart = null;
var data = [
{% for fact in facts|groupby('value') %}
{
label: '{{ fact.grouper.replace("\n", " ") }}',
value: {{ fact.list|length }}
},
{% endfor %}
{
value: 0,
}
]
var fact_values = data.map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];});
var realdata = fact_values.slice(0, 15);
var otherdata = fact_values.slice(15);
if (otherdata.length > 0) {
realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]);
}
{% endif %}
{% endblock javascript %}
{% block onload_script %}
$('table').tablesort();
{% if render_graph %}
chart = c3.generate({
{% macro extra_options(caller) %}
// No per page AJAX
'serverSide': false,
{% endmacro %}
{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, fact=fact, value=value), default_length=config.NORMAL_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }}
{% if render_graph %}
table.on('xhr', function(e, settings, json){
var fact_values = json['chart'].map(function(item) { return [item.label, item.value]; }).filter(function(item){return item[0];}).sort(function(a,b){return b[1] - a[1];});
var realdata = fact_values.slice(0, 15);
var otherdata = fact_values.slice(15);
if (otherdata.length > 0) {
realdata.push(["other", otherdata.reduce(function(a,b){return a + b[1];},0)]);
}
c3.generate({
bindto: '#factChart',
data: {
columns: realdata,
type : '{{config.GRAPH_TYPE|default('pie')}}',
}
});
{% endif %}
})
{% endif %}
{% endblock onload_script %}
{% block content %}
{% if render_graph %}
<div id="factChart" width="300" height="300"></div>
<h1>{{name}}{% if value %}/{{value}}{% endif %} ({{facts|length}})</h1>
{% if value %}
{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, show_value=False, margin_bottom=10)}}
{% else %}
{{macros.facts_table(facts, current_env=current_env, autofocus=True, show_node=True, link_facts=True, margin_bottom=10)}}
{% endif %}
<h1>{{ fact }}{% if value %}/{{ value }}{% endif %}</h1>
<table id="facts_table" class='ui fixed very basic compact table stackable'>
<thead>
<tr>
<th>Node</th>
{% if not value %}<th>Value</th>{% endif %}
</tr>
</thead>
<tbody>
</tbody>
</table>
{% endblock content %}

View File

@@ -4,26 +4,19 @@
<input autofocus="autofocus" class="filter-list" placeholder="Type here to filter...">
</div>
<div class="ui searchable stackable doubling four column grid factlist">
{%- for column in facts_columns %}
<div class="column">
{%- set facts_count = 0 -%}
{%- set break = facts_len//4 + 1 -%}
{%- for key,facts_list in facts_dict %}
{%- for letter in column %}
<div class="ui list_hide_segment segment">
<a class="ui darkblue ribbon label">{{key}}</a>
<a class="ui darkblue ribbon label">{{ letter[0][0]|upper }}</a>
<ul>
{%- for fact in facts_list %}
<li><a href="{{url_for('fact', env=current_env, fact=fact)}}">{{fact}}</a></li>
{%- for fact in letter %}
<li><a href="{{url_for('fact', env=current_env, fact=fact)}}">{{ fact }}</a></li>
{%- endfor %}
</ul>
</div>
{%- set facts_count = facts_count + facts_list|length -%}
{%- if facts_count >= break -%}
</div>
<div class="column">
{%- set break = facts_len//4 + 1 + break -%}
{%- endif -%}
{%- set facts_count = facts_count + 5 -%}
{% endfor %}
{%- endfor %}
</div>
{%- endfor %}
</div>
{% endblock content %}

View File

@@ -17,7 +17,13 @@
'pagingType': 'simple',
"bFilter": false,
{% endmacro %}
{% macro facts_extra_options(caller) %}
'paging': false,
// No per page AJAX
'serverSide': false,
{% endmacro %}
{{ macros.datatable_init(table_html_id="reports_table", ajax_url=url_for('reports_ajax', env=current_env, node_name=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }}
{{ macros.datatable_init(table_html_id="facts_table", ajax_url=url_for('fact_ajax', env=current_env, node=node.name), default_length=config.LITTLE_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=facts_extra_options) }}
{% endblock onload_script %}
{% block content %}
@@ -69,7 +75,16 @@
</div>
<div class='column'>
<h1>Facts</h1>
{{macros.facts_table(facts, link_facts=True, condensed=True, current_env=current_env)}}
<table id="facts_table" class='ui fixed very basic very compact table stackable'>
<thead>
<tr>
<th>Name</th>
<th>Value</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>
</div>
{% endblock content %}

View File

@@ -582,7 +582,7 @@ def test_catalogs_json(client, mocker,
found_status = None
for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']:
val = BeautifulSoup(line[0], 'html.parser').find_all(
'a', {"href": "/node/node-%s/" % status})
'a', {"href": "/node/node-%s" % status})
if len(val) == 1:
found_status = status
break
@@ -609,7 +609,7 @@ def test_catalogs_json_compare(client, mocker,
found_status = None
for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']:
val = BeautifulSoup(line[0], 'html.parser').find_all(
'a', {"href": "/node/node-%s/" % status})
'a', {"href": "/node/node-%s" % status})
if len(val) == 1:
found_status = status
break
@@ -622,47 +622,202 @@ def test_catalogs_json_compare(client, mocker,
assert len(val) == 1
def test_inventory_view(client, mocker, mock_puppetdb_environments):
rv = client.get('/inventory')
assert rv.status_code == 200
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
def test_inventory_json(client, mocker, mock_puppetdb_environments):
facts = ['fqdn', 'ipaddress', 'lsbdistdescription', 'hardwaremodel',
'kernelrelease', 'puppetversion']
values = [
['node-1', 'X.X.X.X', 'os7', 'server', '4.3', 'X.X.X'],
['node-2', 'X.X.X.X', 'os5', 'server', '4.1', 'X.X.X'],
['node-3', 'X.X.X.X', 'os6', 'server', '4.2', 'X.X.X'],
['node-4', 'X.X.X.X', 'os4', 'server', '4.3', 'X.X.X'],
]
query_data = {'facts': []}
query_data['facts'].append([])
for i, value in enumerate(values):
for idx, column in enumerate(facts):
query_data['facts'][0].append({
'certname': value[0],
'name': column,
'value': value[idx],
'environment': 'production'
})
def test_facts_view(client, mocker, mock_puppetdb_environments):
query_data = {
'fact-names': [[chr(i) for i in range(ord('a'), ord('z') + 1)]]
}
dbquery = MockDbQuery(query_data)
mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
rv = client.get('/inventory/json')
rv = client.get('/facts')
assert rv.status_code == 200
result_json = json.loads(rv.data.decode('utf-8'))
assert 'data' in result_json
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
for value in values:
for line in result_json['data']:
if value[0] in line[0]:
assert line[1:] == value[1:]
break
else:
raise Exception("Input %s not found" % value)
searchable = soup.find('div', {'class': 'searchable'})
vals = searchable.find_all('div', {'class': 'column'})
assert len(vals) == 4
def test_fact_view_with_graph(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
rv = client.get('/fact/architecture')
assert rv.status_code == 200
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
vals = soup.find_all('div', {"id": "factChart"})
assert len(vals) == 1
def test_fact_view_without_graph(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
rv = client.get('/%2A/fact/augeas')
assert rv.status_code == 200
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
vals = soup.find_all('div', {"id": "factChart"})
assert len(vals) == 0
def test_fact_value_view(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
rv = client.get('/fact/architecture/amd64')
assert rv.status_code == 200
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
vals = soup.find_all('div', {"id": "factChart"})
assert len(vals) == 0
def test_node_view(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
rv = client.get('/node/node-failed')
assert rv.status_code == 200
soup = BeautifulSoup(rv.data, 'html.parser')
assert soup.title.contents[0] == 'Puppetboard'
vals = soup.find_all('table', {"id": "facts_table"})
assert len(vals) == 1
vals = soup.find_all('table', {"id": "reports_table"})
assert len(vals) == 1
def test_fact_json_with_graph(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
values = ['a', 'b', 'b', 'd', True, 'a\nb']
query_data = {'facts': []}
query_data['facts'].append([])
for i, value in enumerate(values):
query_data['facts'][0].append({
'certname': 'node-%s' % i,
'name': 'architecture',
'value': value,
'environment': 'production'
})
dbquery = MockDbQuery(query_data)
mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
rv = client.get('/fact/architecture/json')
assert rv.status_code == 200
result_json = json.loads(rv.data.decode('utf-8'))
assert 'data' in result_json
assert len(result_json['data']) == 6
for line in result_json['data']:
assert len(line) == 2
assert 'chart' in result_json
assert len(result_json['chart']) == 5
# Test group_by
assert result_json['chart'][0]['value'] == 2
def test_fact_json_without_graph(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
values = ['a', 'b', 'b', 'd']
query_data = {'facts': []}
query_data['facts'].append([])
for i, value in enumerate(values):
query_data['facts'][0].append({
'certname': 'node-%s' % i,
'name': 'architecture',
'value': value,
'environment': 'production'
})
dbquery = MockDbQuery(query_data)
mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
rv = client.get('/%2A/fact/augeas/json')
assert rv.status_code == 200
result_json = json.loads(rv.data.decode('utf-8'))
assert 'data' in result_json
assert len(result_json['data']) == 4
for line in result_json['data']:
assert len(line) == 2
assert 'chart' not in result_json
def test_fact_value_json(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
values = ['a', 'b', 'b', 'd']
query_data = {'facts': []}
query_data['facts'].append([])
for i, value in enumerate(values):
query_data['facts'][0].append({
'certname': 'node-%s' % i,
'name': 'architecture',
'value': value,
'environment': 'production'
})
dbquery = MockDbQuery(query_data)
mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
rv = client.get('/fact/architecture/amd64/json')
assert rv.status_code == 200
result_json = json.loads(rv.data.decode('utf-8'))
assert 'data' in result_json
assert len(result_json['data']) == 4
for line in result_json['data']:
assert len(line) == 1
assert 'chart' not in result_json
def test_node_facts_json(client, mocker,
mock_puppetdb_environments,
mock_puppetdb_default_nodes):
values = ['a', 'b', 'b', 'd']
query_data = {'facts': []}
query_data['facts'].append([])
for i, value in enumerate(values):
query_data['facts'][0].append({
'certname': 'node-failed',
'name': 'fact-%s' % i,
'value': value,
'environment': 'production'
})
dbquery = MockDbQuery(query_data)
mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
rv = client.get('/node/node-failed/facts/json')
assert rv.status_code == 200
result_json = json.loads(rv.data.decode('utf-8'))
assert 'data' in result_json
assert len(result_json['data']) == 4
for line in result_json['data']:
assert len(line) == 2
assert 'chart' not in result_json

View File

@@ -3,7 +3,7 @@ from puppetboard import app, forms
def test_form_valid(capsys):
for form in [forms.QueryForm, forms.CatalogForm]:
for form in [forms.QueryForm]:
with app.app.test_request_context():
qf = form()
out, err = capsys.readouterr()

View File

@@ -2,6 +2,9 @@
envlist = py{26,27,35,36}
[testenv]
deps=
-rrequirements-test.txt
bandit
commands=
py.test --cov=puppetboard --pep8 -v
py{27,35,36}: bandit -r puppetboard