puppet/puppetboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Travis ci integration (#267)

Browse Source 
* Initial travis-ci integration

* Format code base to PEP8
This commit is contained in:
Mike Terzo
2016-07-13 20:59:07 -04:00
committed by Corey Hammerton
parent 0ac64530bf
commit faac5fa1bc
13 changed files with 211 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.coveragerc Normal file
View File

@@ -0,0 +1,3 @@
[report]
exclude_lines =
pragma: notest

7
.gitignore vendored
View File

@@ -1,5 +1,8 @@
*.py[cod]
# Editor tmp files
.*.sw*
# C extensions
*.so
@@ -22,6 +25,7 @@ lib64
pip-log.txt
# Unit test / coverage reports
.cache
.coverage
.tox
nosetests.xml
@@ -36,3 +40,6 @@ nosetests.xml
# PuppetDB Settings
/settings.py
# Virtual Environment
venv

13
.travis.yml Normal file
View File

@@ -0,0 +1,13 @@
language: python
python:
- "2.6"
- "2.7"
install:
- pip install -r requirements.txt
- pip install -r requirements-test.txt
- pip install -q coverage coveralls --use-wheel
script: py.test --cov=puppetboard --pep8 -v
after_success:
- coveralls

2
conftest.py Normal file
View File

@@ -0,0 +1,2 @@
import puppetboard
import test

4
dev.py
View File

@@ -20,12 +20,12 @@ if __name__ == '__main__':
app.config['DEV_COFFEE_LOCATION'], '-w', '-c',
'-o', 'puppetboard/static/js',
'puppetboard/static/coffeescript'
])
])
except OSError:
app.logger.error(
'The coffee executable was not found, disabling automatic '
'CoffeeScript compilation'
)
)
# Start the Flask development server
app.debug = True

212
puppetboard/app.py
View File

@@ -14,7 +14,7 @@ from flask import (
Flask, render_template, abort, url_for,
Response, stream_with_context, redirect,
request, session
)
)
from pypuppetdb import connect
from pypuppetdb.QueryBuilder import *
@@ -23,8 +23,9 @@ from puppetboard.forms import (CatalogForm, QueryForm)
from puppetboard.utils import (
get_or_abort, yield_or_stop,
jsonprint, prettyprint, Pagination
)
)
DEFAULT_ORDER_BY = '[{"field": "start_time", "order": "desc"}]'
app = Flask(__name__)
@@ -59,12 +60,14 @@ def stream_template(template_name, **context):
rv.enable_buffering(5)
return rv
def url_for_field(field, value):
args = request.view_args.copy()
args.update(request.args.copy())
args[field] = value
return url_for(request.endpoint, **args)
def environments():
envs = get_or_abort(puppetdb.environments)
x = []
@@ -74,12 +77,14 @@ def environments():
return x
def check_env(env, envs):
if env != '*' and env not in envs:
abort(404)
app.jinja_env.globals['url_for_field'] = url_for_field
@app.context_processor
def utility_processor():
def now(format='%m/%d/%Y %H:%M:%S'):
@@ -163,7 +168,7 @@ def index(env):
num_nodes_query.add_query(query)
if app.config['OVERVIEW_FILTER'] != None:
query.add(app.config['OVERVIEW_FILTER'])
query.add(app.config['OVERVIEW_FILTER'])
num_resources_query = ExtractOperator()
num_resources_query.add_field(FunctionOperator('count'))
@@ -186,9 +191,9 @@ def index(env):
metrics['avg_resources_node'] = 0
nodes = get_or_abort(puppetdb.nodes,
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
nodes_overview = []
stats = {
@@ -197,7 +202,7 @@ def index(env):
'failed': 0,
'unreported': 0,
'noop': 0
}
}
for node in nodes:
if node.status == 'unreported':
@@ -221,7 +226,7 @@ def index(env):
stats=stats,
envs=envs,
current_env=env
)
)
@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@@ -263,9 +268,9 @@ def nodes(env):
nodes.append(node)
return Response(stream_with_context(
stream_template('nodes.html',
nodes=nodes,
envs=envs,
current_env=env)))
nodes=nodes,
envs=envs,
current_env=env)))
@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@@ -286,28 +291,28 @@ def inventory(env):
envs = environments()
check_env(env, envs)
fact_desc = [] # a list of fact descriptions to go
# in the table header
fact_desc = [] # a list of fact descriptions to go
# in the table header
fact_names = [] # a list of inventory fact names
factvalues = {} # values of the facts for all the nodes
# indexed by node name and fact name
nodedata = {} # a dictionary containing list of inventoried
# facts indexed by node name
nodelist = set() # a set of node names
# indexed by node name and fact name
nodedata = {} # a dictionary containing list of inventoried
# facts indexed by node name
nodelist = set() # a set of node names
# load the list of items/facts we want in our inventory
try:
inv_facts = app.config['INVENTORY_FACTS']
except KeyError:
inv_facts = [ ('Hostname' ,'fqdn' ),
('IP Address' ,'ipaddress' ),
('OS' ,'lsbdistdescription'),
('Architecture' ,'hardwaremodel' ),
('Kernel Version','kernelrelease' ) ]
inv_facts = [('Hostname', 'fqdn'),
('IP Address', 'ipaddress'),
('OS', 'lsbdistdescription'),
('Architecture', 'hardwaremodel'),
('Kernel Version', 'kernelrelease')]
# generate a list of descriptions and a list of fact names
# from the list of tuples inv_facts.
for description,name in inv_facts:
for description, name in inv_facts:
fact_desc.append(description)
fact_names.append(name)
@@ -325,7 +330,7 @@ def inventory(env):
# convert the json in easy to access data structure
for fact in facts:
factvalues[fact.node,fact.name] = fact.value
factvalues[fact.node, fact.name] = fact.value
nodelist.add(fact.node)
# generate the per-host data
@@ -333,19 +338,20 @@ def inventory(env):
nodedata[node] = []
for fact_name in fact_names:
try:
nodedata[node].append(factvalues[node,fact_name])
nodedata[node].append(factvalues[node, fact_name])
except KeyError:
nodedata[node].append("undef")
return Response(stream_with_context(
stream_template('inventory.html',
nodedata=nodedata,
fact_desc=fact_desc,
envs=envs,
current_env=env)))
nodedata=nodedata,
fact_desc=fact_desc,
envs=envs,
current_env=env)))
@app.route('/node/<node_name>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/node/<node_name>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/node/<node_name>')
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
@@ -367,9 +373,9 @@ def node(env, node_name):
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
query=query,
limit=app.config['REPORTS_COUNT'],
order_by=DEFAULT_ORDER_BY)
reports, reports_events = tee(reports)
report_event_counts = {}
@@ -408,7 +414,8 @@ def node(env, node_name):
current_env=env)
@app.route('/reports/', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1})
@app.route('/reports/',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1})
@app.route('/<env>/reports/', defaults={'page': 1})
@app.route('/<env>/reports/page/<int:page>')
def reports(env, page):
@@ -435,17 +442,17 @@ def reports(env, page):
try:
paging_args = {'limit': int(limit)}
paging_args['offset'] = int((page-1) * paging_args['limit'])
paging_args['offset'] = int((page - 1) * paging_args['limit'])
except ValueError:
paging_args = {}
reports = get_or_abort(puppetdb.reports,
query=reports_query,
order_by='[{"field": "start_time", "order": "desc"}]',
**paging_args)
query=reports_query,
order_by=DEFAULT_ORDER_BY,
**paging_args)
total = get_or_abort(puppetdb._query,
'reports',
query=total_query)
'reports',
query=total_query)
total = total[0]['count']
reports, reports_events = tee(reports)
report_event_counts = {}
@@ -488,7 +495,8 @@ def reports(env, page):
limit=paging_args.get('limit', total))))
@app.route('/reports/<node_name>/', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1})
@app.route('/reports/<node_name>/',
defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1})
@app.route('/<env>/reports/<node_name>', defaults={'page': 1})
@app.route('/<env>/reports/<node_name>/page/<int:page>')
def reports_node(env, node_name, page):
@@ -517,13 +525,13 @@ def reports_node(env, node_name, page):
total_query.add_query(query)
reports = get_or_abort(puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
offset=(page-1) * app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
query=query,
limit=app.config['REPORTS_COUNT'],
offset=(page - 1) * app.config['REPORTS_COUNT'],
order_by=DEFAULT_ORDER_BY)
total = get_or_abort(puppetdb._query,
'reports',
query=total_query)
'reports',
query=total_query)
total = total[0]['count']
reports, reports_events = tee(reports)
report_event_counts = {}
@@ -565,7 +573,8 @@ def reports_node(env, node_name, page):
current_env=env)
@app.route('/report/<node_name>/<report_id>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/report/<node_name>/<report_id>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/report/<node_name>/<report_id>')
def report(env, node_name, report_id):
"""Displays a single report including all the events associated with that
@@ -637,10 +646,11 @@ def facts(env):
sorted_facts_dict = sorted(facts_dict.items())
return render_template('facts.html',
facts_dict=sorted_facts_dict,
facts_len=sum(map(len,facts_dict.values())) + len(facts_dict)*5,
envs=envs,
current_env=env)
facts_dict=sorted_facts_dict,
facts_len=(sum(map(len, facts_dict.values())) +
len(facts_dict) * 5),
envs=envs,
current_env=env)
@app.route('/fact/<fact>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@@ -679,7 +689,8 @@ def fact(env, fact):
current_env=env)))
@app.route('/fact/<fact>/<value>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/fact/<fact>/<value>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/fact/<fact>/<value>')
def fact_value(env, fact, value):
"""On asking for fact/value get all nodes with that fact.
@@ -700,9 +711,9 @@ def fact_value(env, fact, value):
query = EqualsOperator("environment", env)
facts = get_or_abort(puppetdb.facts,
name=fact,
value=value,
query=query)
name=fact,
value=value,
query=query)
localfacts = [f for f in yield_or_stop(facts)]
return render_template(
'fact.html',
@@ -713,7 +724,8 @@ def fact_value(env, fact, value):
current_env=env)
@app.route('/query', methods=('GET', 'POST'), defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/query', methods=('GET', 'POST'),
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/query', methods=('GET', 'POST'))
def query(env):
"""Allows to execute raw, user created querries against PuppetDB. This is
@@ -745,14 +757,14 @@ def query(env):
form.endpoints.data,
query=query)
return render_template('query.html',
form=form,
result=result,
envs=envs,
current_env=env)
form=form,
result=result,
envs=envs,
current_env=env)
return render_template('query.html',
form=form,
envs=envs,
current_env=env)
form=form,
envs=envs,
current_env=env)
else:
log.warn('Access to query interface disabled by administrator..')
abort(403)
@@ -772,12 +784,13 @@ def metrics(env):
metrics = get_or_abort(puppetdb._query, 'mbean')
return render_template('metrics.html',
metrics=sorted(metrics.keys()),
envs=envs,
current_env=env)
metrics=sorted(metrics.keys()),
envs=envs,
current_env=env)
@app.route('/metric/<metric>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/metric/<metric>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/metric/<metric>')
def metric(env, metric):
"""Lists all information about the metric of the given name.
@@ -798,6 +811,7 @@ def metric(env, metric):
envs=envs,
current_env=env)
@app.route('/catalogs', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/catalogs')
def catalogs(env):
@@ -819,10 +833,11 @@ def catalogs(env):
query.add(NullOperator("catalog_timestamp", False))
order_by_str = '[{"field": "certname", "order": "asc"}]'
nodes = get_or_abort(puppetdb.nodes,
query=query,
with_status=False,
order_by='[{"field": "certname", "order": "asc"}]')
query=query,
with_status=False,
order_by=oder_by_str)
nodes, temp = tee(nodes)
for node in temp:
@@ -855,7 +870,9 @@ def catalogs(env):
log.warn('Access to catalog interface disabled by administrator')
abort(403)
@app.route('/catalog/<node_name>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/catalog/<node_name>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/catalog/<node_name>')
def catalog_node(env, node_name):
"""Fetches from PuppetDB the compiled catalog of a given node.
@@ -868,16 +885,18 @@ def catalog_node(env, node_name):
if app.config['ENABLE_CATALOG']:
catalog = get_or_abort(puppetdb.catalog,
node=node_name)
node=node_name)
return render_template('catalog.html',
catalog=catalog,
envs=envs,
current_env=env)
catalog=catalog,
envs=envs,
current_env=env)
else:
log.warn('Access to catalog interface disabled by administrator')
abort(403)
@app.route('/catalog/submit', methods=['POST'], defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/catalog/submit', methods=['POST'],
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/catalog/submit', methods=['POST'])
def catalog_submit(env):
"""Receives the submitted form data from the catalogs page and directs
@@ -901,15 +920,17 @@ def catalog_submit(env):
against = form.against.data
return redirect(
url_for('catalog_compare',
env=env,
compare=compare,
against=against))
env=env,
compare=compare,
against=against))
return redirect(url_for('catalogs', env=env))
else:
log.warn('Access to catalog interface disabled by administrator')
abort(403)
@app.route('/catalogs/compare/<compare>...<against>', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/catalogs/compare/<compare>...<against>',
defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/catalogs/compare/<compare>...<against>')
def catalog_compare(env, compare, against):
"""Compares the catalog of one node, parameter compare, with that of
@@ -923,19 +944,20 @@ def catalog_compare(env, compare, against):
if app.config['ENABLE_CATALOG']:
compare_cat = get_or_abort(puppetdb.catalog,
node=compare)
node=compare)
against_cat = get_or_abort(puppetdb.catalog,
node=against)
node=against)
return render_template('catalog_compare.html',
compare=compare_cat,
against=against_cat,
envs=envs,
current_env=env)
compare=compare_cat,
against=against_cat,
envs=envs,
current_env=env)
else:
log.warn('Access to catalog interface disabled by administrator')
abort(403)
@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@app.route('/<env>/radiator')
def radiator(env):
@@ -966,13 +988,11 @@ def radiator(env):
query=metric_query)
num_nodes = metrics[0]['count']
nodes = puppetdb.nodes(
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True
)
)
stats = {
'changed_percent': 0,
@@ -989,8 +1009,6 @@ def radiator(env):
'unreported': 0,
}
for node in nodes:
if node.status == 'unreported':
stats['unreported'] += 1
@@ -1001,18 +1019,18 @@ def radiator(env):
elif node.status == 'noop':
stats['noop'] += 1
elif node.status == 'skipped':
stats['skipped'] +=1
stats['skipped'] += 1
else:
stats['unchanged'] += 1
stats['changed_percent'] = int(100 * stats['changed'] / float(num_nodes))
stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes))
stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes))
stats['skipped_percent'] = int(100 * stats['skipped'] / float(num_nodes))
stats['unchanged_percent'] = int(100 * stats['unchanged'] / float(num_nodes))
stats['unreported_percent'] = int(100 * stats['unreported'] / float(num_nodes))
stats['unchanged_percent'] = int(100 * (stats['unchanged'] /
float(num_nodes)))
stats['unreported_percent'] = int(100 * (stats['unreported'] /
float(num_nodes)))
return render_template(
'radiator.html',

12
puppetboard/default_settings.py
View File

@@ -31,10 +31,10 @@ GRAPH_FACTS = ['architecture',
'osfamily',
'puppetversion',
'processorcount']
INVENTORY_FACTS = [ ('Hostname', 'fqdn' ),
('IP Address', 'ipaddress' ),
('OS', 'lsbdistdescription'),
('Architecture', 'hardwaremodel' ),
('Kernel Version', 'kernelrelease' ),
('Puppet Version', 'puppetversion' ), ]
INVENTORY_FACTS = [('Hostname', 'fqdn'),
('IP Address', 'ipaddress'),
('OS', 'lsbdistdescription'),
('Architecture', 'hardwaremodel'),
('Kernel Version', 'kernelrelease'),
('Puppet Version', 'puppetversion'), ]
REFRESH_RATE = 30

3
puppetboard/forms.py
View File

@@ -26,9 +26,10 @@ class QueryForm(Form):
('edges', 'Edges'),
('environments', 'Environments'),
('pql', 'PQL'),
])
])
rawjson = BooleanField('Raw JSON')
class CatalogForm(Form):
"""The form used to compare the catalogs of different nodes."""
compare = HiddenField('compare')

29
puppetboard/utils.py
View File

@@ -19,40 +19,44 @@ except NameError:
log = logging.getLogger(__name__)
def jsonprint(value):
return json.dumps(value, indent=2, separators=(',', ': '))
def formatvalue(value):
if isinstance(value, str):
return value
return value
elif isinstance(value, list):
return ", ".join(value)
return ", ".join(value)
elif isinstance(value, dict):
ret = ""
for k in value:
ret += k+" => "+formatvalue(value[k])+",<br/>"
return ret
ret = ""
for k in value:
ret += k + " => " + formatvalue(value[k]) + ",<br/>"
return ret
else:
return str(value)
return str(value)
def prettyprint(value):
html = '<table class="ui basic fixed sortable table"><thead><tr>'
# Get keys
for k in value[0]:
html += "<th>"+k+"</th>"
html += "<th>" + k + "</th>"
html += "</tr></thead><tbody>"
for e in value:
html += "<tr>"
for k in e:
html += "<td>"+formatvalue(e[k])+"</td>"
html += "<td>" + formatvalue(e[k]) + "</td>"
html += "</tr>"
html += "</tbody></table>"
return(html)
def get_or_abort(func, *args, **kwargs):
"""Execute the function with its arguments and handle the possible
errors that might occur.
@@ -87,6 +91,7 @@ def yield_or_stop(generator):
except (EmptyResponseError, ConnectionError, HTTPError):
raise StopIteration
class Pagination(object):
def __init__(self, page, per_page, total_count):
@@ -111,9 +116,9 @@ class Pagination(object):
last = 0
for num in xrange(1, self.pages + 1):
if num <= left_edge or \
(num > self.page - left_current - 1 and \
num < self.page + right_current) or \
num > self.pages - right_edge:
(num > self.page - left_current - 1 and
num < self.page + right_current) or \
num > self.pages - right_edge:
if last + 1 != num:
yield None
yield num

6
requirements-test.txt Normal file
View File

@@ -0,0 +1,6 @@
pep8==1.6.2
coverage==4.0
mock==1.3.0
pytest-pep8==1.0.5
pytest-cov==2.2.1
cov-core==1.15.0

17
setup.cfg
View File

@@ -6,3 +6,20 @@ build_requires = python-setuptools
requires = python-flask
python-flask-wtf
python-pypuppetdb
[pep8]
max-line-length=100
exclude=venv,dist,build
ignore=E402
[nosetests]
with-coverage = 1
with-xunit = 1
cover-package = puppetboard
[flake8]
exclude=venv
[pytest]
addopts = --cov=puppetboard --cov-report=term-missing
norecursedirs = docs .tox venv
pep8ignore = E402

4
setup.py
View File

@@ -33,7 +33,7 @@ setup(
"Flask-WTF >= 0.12, <= 0.13",
"WTForms >= 2.0, < 3.0",
"pypuppetdb >= 0.3.0, < 0.4.0",
],
],
keywords="puppet puppetdb puppetboard",
classifiers=[
'Development Status :: 3 - Alpha',
@@ -49,5 +49,5 @@ setup(
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.2',
'Programming Language :: Python :: 3.3',
],
],
)

19
test/test_app.py Normal file
View File

@@ -0,0 +1,19 @@
import os
from puppetboard import app
import unittest
import tempfile
class AppTestCase(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_first_test(self):
self.assertTrue(True)
if __name__ == '__main__':
unittest.main()