diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..4f627e1 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[report] +exclude_lines = + pragma: notest diff --git a/.gitignore b/.gitignore index f7aa916..80377f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ *.py[cod] +# Editor tmp files +.*.sw* + # C extensions *.so @@ -22,6 +25,7 @@ lib64 pip-log.txt # Unit test / coverage reports +.cache .coverage .tox nosetests.xml @@ -36,3 +40,6 @@ nosetests.xml # PuppetDB Settings /settings.py + +# Virtual Environment +venv diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..897cc21 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python + +python: + - "2.6" + - "2.7" +install: + - pip install -r requirements.txt + - pip install -r requirements-test.txt + - pip install -q coverage coveralls --use-wheel +script: py.test --cov=puppetboard --pep8 -v + +after_success: + - coveralls diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..ea02241 --- /dev/null +++ b/conftest.py @@ -0,0 +1,2 @@ +import puppetboard +import test diff --git a/dev.py b/dev.py index 7576fa1..36df91d 100644 --- a/dev.py +++ b/dev.py @@ -20,12 +20,12 @@ if __name__ == '__main__': app.config['DEV_COFFEE_LOCATION'], '-w', '-c', '-o', 'puppetboard/static/js', 'puppetboard/static/coffeescript' - ]) + ]) except OSError: app.logger.error( 'The coffee executable was not found, disabling automatic ' 'CoffeeScript compilation' - ) + ) # Start the Flask development server app.debug = True diff --git a/puppetboard/app.py b/puppetboard/app.py index bf43d9b..a3d74ab 100644 --- a/puppetboard/app.py +++ b/puppetboard/app.py @@ -14,7 +14,7 @@ from flask import ( Flask, render_template, abort, url_for, Response, stream_with_context, redirect, request, session - ) +) from pypuppetdb import connect from pypuppetdb.QueryBuilder import * @@ -23,8 +23,9 @@ from puppetboard.forms import (CatalogForm, QueryForm) from puppetboard.utils import ( get_or_abort, yield_or_stop, jsonprint, prettyprint, Pagination - ) +) +DEFAULT_ORDER_BY = '[{"field": "start_time", "order": "desc"}]' app = Flask(__name__) @@ -59,12 +60,14 @@ def stream_template(template_name, **context): rv.enable_buffering(5) return rv + def url_for_field(field, value): args = request.view_args.copy() args.update(request.args.copy()) args[field] = value return url_for(request.endpoint, **args) + def environments(): envs = get_or_abort(puppetdb.environments) x = [] @@ -74,12 +77,14 @@ def environments(): return x + def check_env(env, envs): if env != '*' and env not in envs: abort(404) app.jinja_env.globals['url_for_field'] = url_for_field + @app.context_processor def utility_processor(): def now(format='%m/%d/%Y %H:%M:%S'): @@ -163,7 +168,7 @@ def index(env): num_nodes_query.add_query(query) if app.config['OVERVIEW_FILTER'] != None: - query.add(app.config['OVERVIEW_FILTER']) + query.add(app.config['OVERVIEW_FILTER']) num_resources_query = ExtractOperator() num_resources_query.add_field(FunctionOperator('count')) @@ -186,9 +191,9 @@ def index(env): metrics['avg_resources_node'] = 0 nodes = get_or_abort(puppetdb.nodes, - query=query, - unreported=app.config['UNRESPONSIVE_HOURS'], - with_status=True) + query=query, + unreported=app.config['UNRESPONSIVE_HOURS'], + with_status=True) nodes_overview = [] stats = { @@ -197,7 +202,7 @@ def index(env): 'failed': 0, 'unreported': 0, 'noop': 0 - } + } for node in nodes: if node.status == 'unreported': @@ -221,7 +226,7 @@ def index(env): stats=stats, envs=envs, current_env=env - ) + ) @app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @@ -263,9 +268,9 @@ def nodes(env): nodes.append(node) return Response(stream_with_context( stream_template('nodes.html', - nodes=nodes, - envs=envs, - current_env=env))) + nodes=nodes, + envs=envs, + current_env=env))) @app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @@ -286,28 +291,28 @@ def inventory(env): envs = environments() check_env(env, envs) - fact_desc = [] # a list of fact descriptions to go - # in the table header + fact_desc = [] # a list of fact descriptions to go + # in the table header fact_names = [] # a list of inventory fact names factvalues = {} # values of the facts for all the nodes - # indexed by node name and fact name - nodedata = {} # a dictionary containing list of inventoried - # facts indexed by node name - nodelist = set() # a set of node names + # indexed by node name and fact name + nodedata = {} # a dictionary containing list of inventoried + # facts indexed by node name + nodelist = set() # a set of node names # load the list of items/facts we want in our inventory try: inv_facts = app.config['INVENTORY_FACTS'] except KeyError: - inv_facts = [ ('Hostname' ,'fqdn' ), - ('IP Address' ,'ipaddress' ), - ('OS' ,'lsbdistdescription'), - ('Architecture' ,'hardwaremodel' ), - ('Kernel Version','kernelrelease' ) ] + inv_facts = [('Hostname', 'fqdn'), + ('IP Address', 'ipaddress'), + ('OS', 'lsbdistdescription'), + ('Architecture', 'hardwaremodel'), + ('Kernel Version', 'kernelrelease')] # generate a list of descriptions and a list of fact names # from the list of tuples inv_facts. - for description,name in inv_facts: + for description, name in inv_facts: fact_desc.append(description) fact_names.append(name) @@ -325,7 +330,7 @@ def inventory(env): # convert the json in easy to access data structure for fact in facts: - factvalues[fact.node,fact.name] = fact.value + factvalues[fact.node, fact.name] = fact.value nodelist.add(fact.node) # generate the per-host data @@ -333,19 +338,20 @@ def inventory(env): nodedata[node] = [] for fact_name in fact_names: try: - nodedata[node].append(factvalues[node,fact_name]) + nodedata[node].append(factvalues[node, fact_name]) except KeyError: nodedata[node].append("undef") return Response(stream_with_context( stream_template('inventory.html', - nodedata=nodedata, - fact_desc=fact_desc, - envs=envs, - current_env=env))) + nodedata=nodedata, + fact_desc=fact_desc, + envs=envs, + current_env=env))) -@app.route('/node/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('/node/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//node/') def node(env, node_name): """Display a dashboard for a node showing as much data as we have on that @@ -367,9 +373,9 @@ def node(env, node_name): node = get_or_abort(puppetdb.node, node_name) facts = node.facts() reports = get_or_abort(puppetdb.reports, - query=query, - limit=app.config['REPORTS_COUNT'], - order_by='[{"field": "start_time", "order": "desc"}]') + query=query, + limit=app.config['REPORTS_COUNT'], + order_by=DEFAULT_ORDER_BY) reports, reports_events = tee(reports) report_event_counts = {} @@ -408,7 +414,8 @@ def node(env, node_name): current_env=env) -@app.route('/reports/', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1}) +@app.route('/reports/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1}) @app.route('//reports/', defaults={'page': 1}) @app.route('//reports/page/') def reports(env, page): @@ -435,17 +442,17 @@ def reports(env, page): try: paging_args = {'limit': int(limit)} - paging_args['offset'] = int((page-1) * paging_args['limit']) + paging_args['offset'] = int((page - 1) * paging_args['limit']) except ValueError: paging_args = {} reports = get_or_abort(puppetdb.reports, - query=reports_query, - order_by='[{"field": "start_time", "order": "desc"}]', - **paging_args) + query=reports_query, + order_by=DEFAULT_ORDER_BY, + **paging_args) total = get_or_abort(puppetdb._query, - 'reports', - query=total_query) + 'reports', + query=total_query) total = total[0]['count'] reports, reports_events = tee(reports) report_event_counts = {} @@ -488,7 +495,8 @@ def reports(env, page): limit=paging_args.get('limit', total)))) -@app.route('/reports//', defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1}) +@app.route('/reports//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'page': 1}) @app.route('//reports/', defaults={'page': 1}) @app.route('//reports//page/') def reports_node(env, node_name, page): @@ -517,13 +525,13 @@ def reports_node(env, node_name, page): total_query.add_query(query) reports = get_or_abort(puppetdb.reports, - query=query, - limit=app.config['REPORTS_COUNT'], - offset=(page-1) * app.config['REPORTS_COUNT'], - order_by='[{"field": "start_time", "order": "desc"}]') + query=query, + limit=app.config['REPORTS_COUNT'], + offset=(page - 1) * app.config['REPORTS_COUNT'], + order_by=DEFAULT_ORDER_BY) total = get_or_abort(puppetdb._query, - 'reports', - query=total_query) + 'reports', + query=total_query) total = total[0]['count'] reports, reports_events = tee(reports) report_event_counts = {} @@ -565,7 +573,8 @@ def reports_node(env, node_name, page): current_env=env) -@app.route('/report//', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('/report//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//report//') def report(env, node_name, report_id): """Displays a single report including all the events associated with that @@ -637,10 +646,11 @@ def facts(env): sorted_facts_dict = sorted(facts_dict.items()) return render_template('facts.html', - facts_dict=sorted_facts_dict, - facts_len=sum(map(len,facts_dict.values())) + len(facts_dict)*5, - envs=envs, - current_env=env) + facts_dict=sorted_facts_dict, + facts_len=(sum(map(len, facts_dict.values())) + + len(facts_dict) * 5), + envs=envs, + current_env=env) @app.route('/fact/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @@ -679,7 +689,8 @@ def fact(env, fact): current_env=env))) -@app.route('/fact//', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('/fact//', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//fact//') def fact_value(env, fact, value): """On asking for fact/value get all nodes with that fact. @@ -700,9 +711,9 @@ def fact_value(env, fact, value): query = EqualsOperator("environment", env) facts = get_or_abort(puppetdb.facts, - name=fact, - value=value, - query=query) + name=fact, + value=value, + query=query) localfacts = [f for f in yield_or_stop(facts)] return render_template( 'fact.html', @@ -713,7 +724,8 @@ def fact_value(env, fact, value): current_env=env) -@app.route('/query', methods=('GET', 'POST'), defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('/query', methods=('GET', 'POST'), + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//query', methods=('GET', 'POST')) def query(env): """Allows to execute raw, user created querries against PuppetDB. This is @@ -745,14 +757,14 @@ def query(env): form.endpoints.data, query=query) return render_template('query.html', - form=form, - result=result, - envs=envs, - current_env=env) + form=form, + result=result, + envs=envs, + current_env=env) return render_template('query.html', - form=form, - envs=envs, - current_env=env) + form=form, + envs=envs, + current_env=env) else: log.warn('Access to query interface disabled by administrator..') abort(403) @@ -772,12 +784,13 @@ def metrics(env): metrics = get_or_abort(puppetdb._query, 'mbean') return render_template('metrics.html', - metrics=sorted(metrics.keys()), - envs=envs, - current_env=env) + metrics=sorted(metrics.keys()), + envs=envs, + current_env=env) -@app.route('/metric/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) +@app.route('/metric/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//metric/') def metric(env, metric): """Lists all information about the metric of the given name. @@ -798,6 +811,7 @@ def metric(env, metric): envs=envs, current_env=env) + @app.route('/catalogs', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//catalogs') def catalogs(env): @@ -819,10 +833,11 @@ def catalogs(env): query.add(NullOperator("catalog_timestamp", False)) + order_by_str = '[{"field": "certname", "order": "asc"}]' nodes = get_or_abort(puppetdb.nodes, - query=query, - with_status=False, - order_by='[{"field": "certname", "order": "asc"}]') + query=query, + with_status=False, + order_by=oder_by_str) nodes, temp = tee(nodes) for node in temp: @@ -855,7 +870,9 @@ def catalogs(env): log.warn('Access to catalog interface disabled by administrator') abort(403) -@app.route('/catalog/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) + +@app.route('/catalog/', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//catalog/') def catalog_node(env, node_name): """Fetches from PuppetDB the compiled catalog of a given node. @@ -868,16 +885,18 @@ def catalog_node(env, node_name): if app.config['ENABLE_CATALOG']: catalog = get_or_abort(puppetdb.catalog, - node=node_name) + node=node_name) return render_template('catalog.html', - catalog=catalog, - envs=envs, - current_env=env) + catalog=catalog, + envs=envs, + current_env=env) else: log.warn('Access to catalog interface disabled by administrator') abort(403) -@app.route('/catalog/submit', methods=['POST'], defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) + +@app.route('/catalog/submit', methods=['POST'], + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//catalog/submit', methods=['POST']) def catalog_submit(env): """Receives the submitted form data from the catalogs page and directs @@ -901,15 +920,17 @@ def catalog_submit(env): against = form.against.data return redirect( url_for('catalog_compare', - env=env, - compare=compare, - against=against)) + env=env, + compare=compare, + against=against)) return redirect(url_for('catalogs', env=env)) else: log.warn('Access to catalog interface disabled by administrator') abort(403) -@app.route('/catalogs/compare/...', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) + +@app.route('/catalogs/compare/...', + defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//catalogs/compare/...') def catalog_compare(env, compare, against): """Compares the catalog of one node, parameter compare, with that of @@ -923,19 +944,20 @@ def catalog_compare(env, compare, against): if app.config['ENABLE_CATALOG']: compare_cat = get_or_abort(puppetdb.catalog, - node=compare) + node=compare) against_cat = get_or_abort(puppetdb.catalog, - node=against) + node=against) return render_template('catalog_compare.html', - compare=compare_cat, - against=against_cat, - envs=envs, - current_env=env) + compare=compare_cat, + against=against_cat, + envs=envs, + current_env=env) else: log.warn('Access to catalog interface disabled by administrator') abort(403) + @app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']}) @app.route('//radiator') def radiator(env): @@ -966,13 +988,11 @@ def radiator(env): query=metric_query) num_nodes = metrics[0]['count'] - nodes = puppetdb.nodes( query=query, unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True - ) - + ) stats = { 'changed_percent': 0, @@ -989,8 +1009,6 @@ def radiator(env): 'unreported': 0, } - - for node in nodes: if node.status == 'unreported': stats['unreported'] += 1 @@ -1001,18 +1019,18 @@ def radiator(env): elif node.status == 'noop': stats['noop'] += 1 elif node.status == 'skipped': - stats['skipped'] +=1 + stats['skipped'] += 1 else: stats['unchanged'] += 1 - stats['changed_percent'] = int(100 * stats['changed'] / float(num_nodes)) stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) stats['skipped_percent'] = int(100 * stats['skipped'] / float(num_nodes)) - stats['unchanged_percent'] = int(100 * stats['unchanged'] / float(num_nodes)) - stats['unreported_percent'] = int(100 * stats['unreported'] / float(num_nodes)) - + stats['unchanged_percent'] = int(100 * (stats['unchanged'] / + float(num_nodes))) + stats['unreported_percent'] = int(100 * (stats['unreported'] / + float(num_nodes))) return render_template( 'radiator.html', diff --git a/puppetboard/default_settings.py b/puppetboard/default_settings.py index 595e690..60947a2 100644 --- a/puppetboard/default_settings.py +++ b/puppetboard/default_settings.py @@ -31,10 +31,10 @@ GRAPH_FACTS = ['architecture', 'osfamily', 'puppetversion', 'processorcount'] -INVENTORY_FACTS = [ ('Hostname', 'fqdn' ), - ('IP Address', 'ipaddress' ), - ('OS', 'lsbdistdescription'), - ('Architecture', 'hardwaremodel' ), - ('Kernel Version', 'kernelrelease' ), - ('Puppet Version', 'puppetversion' ), ] +INVENTORY_FACTS = [('Hostname', 'fqdn'), + ('IP Address', 'ipaddress'), + ('OS', 'lsbdistdescription'), + ('Architecture', 'hardwaremodel'), + ('Kernel Version', 'kernelrelease'), + ('Puppet Version', 'puppetversion'), ] REFRESH_RATE = 30 diff --git a/puppetboard/forms.py b/puppetboard/forms.py index a5cff3c..7628be0 100644 --- a/puppetboard/forms.py +++ b/puppetboard/forms.py @@ -26,9 +26,10 @@ class QueryForm(Form): ('edges', 'Edges'), ('environments', 'Environments'), ('pql', 'PQL'), - ]) + ]) rawjson = BooleanField('Raw JSON') + class CatalogForm(Form): """The form used to compare the catalogs of different nodes.""" compare = HiddenField('compare') diff --git a/puppetboard/utils.py b/puppetboard/utils.py index 0c73732..53234c9 100644 --- a/puppetboard/utils.py +++ b/puppetboard/utils.py @@ -19,40 +19,44 @@ except NameError: log = logging.getLogger(__name__) + def jsonprint(value): return json.dumps(value, indent=2, separators=(',', ': ')) + def formatvalue(value): if isinstance(value, str): - return value + return value elif isinstance(value, list): - return ", ".join(value) + return ", ".join(value) elif isinstance(value, dict): - ret = "" - for k in value: - ret += k+" => "+formatvalue(value[k])+",
" - return ret + ret = "" + for k in value: + ret += k + " => " + formatvalue(value[k]) + ",
" + return ret else: - return str(value) + return str(value) + def prettyprint(value): html = '' # Get keys for k in value[0]: - html += "" + html += "" html += "" for e in value: html += "" for k in e: - html += "" + html += "" html += "" html += "
"+k+"" + k + "
"+formatvalue(e[k])+"" + formatvalue(e[k]) + "
" return(html) + def get_or_abort(func, *args, **kwargs): """Execute the function with its arguments and handle the possible errors that might occur. @@ -87,6 +91,7 @@ def yield_or_stop(generator): except (EmptyResponseError, ConnectionError, HTTPError): raise StopIteration + class Pagination(object): def __init__(self, page, per_page, total_count): @@ -111,9 +116,9 @@ class Pagination(object): last = 0 for num in xrange(1, self.pages + 1): if num <= left_edge or \ - (num > self.page - left_current - 1 and \ - num < self.page + right_current) or \ - num > self.pages - right_edge: + (num > self.page - left_current - 1 and + num < self.page + right_current) or \ + num > self.pages - right_edge: if last + 1 != num: yield None yield num diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..e4c1a9b --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,6 @@ +pep8==1.6.2 +coverage==4.0 +mock==1.3.0 +pytest-pep8==1.0.5 +pytest-cov==2.2.1 +cov-core==1.15.0 diff --git a/setup.cfg b/setup.cfg index 4dbc01f..820c370 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,20 @@ build_requires = python-setuptools requires = python-flask python-flask-wtf python-pypuppetdb +[pep8] +max-line-length=100 +exclude=venv,dist,build +ignore=E402 + +[nosetests] +with-coverage = 1 +with-xunit = 1 +cover-package = puppetboard + +[flake8] +exclude=venv + +[pytest] +addopts = --cov=puppetboard --cov-report=term-missing +norecursedirs = docs .tox venv +pep8ignore = E402 diff --git a/setup.py b/setup.py index d9d6b03..7a24314 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( "Flask-WTF >= 0.12, <= 0.13", "WTForms >= 2.0, < 3.0", "pypuppetdb >= 0.3.0, < 0.4.0", - ], + ], keywords="puppet puppetdb puppetboard", classifiers=[ 'Development Status :: 3 - Alpha', @@ -49,5 +49,5 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', - ], + ], ) diff --git a/test/test_app.py b/test/test_app.py new file mode 100644 index 0000000..c25399e --- /dev/null +++ b/test/test_app.py @@ -0,0 +1,19 @@ +import os +from puppetboard import app +import unittest +import tempfile + + +class AppTestCase(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_first_test(self): + self.assertTrue(True) + + +if __name__ == '__main__': + unittest.main()