puppet/puppetboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implementing new pypuppetdb.QueryBuilder (#251)

Browse Source 
* puppetboard/app.py: Implementing and utilizing the new PyPuppetDB QueryBuilder

This fixes https://github.com/voxpupuli/puppetboard/issues/239

Replacing all the directly declared query strings with various objects
from the pypuppetdb.QueryBuilder sub-module. Using this Object-Oriented
functionality it is programmatically safer to construct large, complex
queries.

* puppetboard/app.py: Simplifying the environment logic in `node()`

The query in this function will now always be an `AndOperator()` object.
If a specific environment is queried then that constraint is added.

The resulting query difference will be:

env == 'production'

`["and",["=", "environment", "production"],["=", "certname", "puppet01.hammertime.local"]]`

env == '*'

`["and",["=", "certname", "puppet01.hammertime.local"]]`

* puppetboard/app.py: Minor code simplification for more accurate results.

In index() adding the configured OVERVIEW_FILTER query after adding the previous
constraints to num_nodes_query
In inventory() wrapping fact_query in an AndOperator() regardless of environment.
This update makes it more common with other endpoints that only add environment
constraints if an environment is selected.

* requirements.txt: Bumping the pypuppetdb version requirement

* requirements.txt: Bumping pypuppetdb version to new requirement.

* puppetboard/app.py: Fixing module load error.
This commit is contained in:
Corey Hammerton
2016-06-23 20:26:21 -04:00
committed by GitHub
parent 39f48c185c
commit adac2a46bf
2 changed files with 83 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
puppetboard/app.py
View File

@@ -17,6 +17,7 @@ from flask import (
) )
from pypuppetdb import connect from pypuppetdb import connect
from pypuppetdb.QueryBuilder import *
from puppetboard.forms import (CatalogForm, QueryForm) from puppetboard.forms import (CatalogForm, QueryForm)
from puppetboard.utils import ( from puppetboard.utils import (
@@ -83,7 +84,7 @@ app.jinja_env.globals['url_for_field'] = url_for_field
def utility_processor(): def utility_processor():
def now(format='%m/%d/%Y %H:%M:%S'): def now(format='%m/%d/%Y %H:%M:%S'):
"""returns the formated datetime""" """returns the formated datetime"""
return datetime.now().strftime(format) return datetime.datetime.now().strftime(format)
return dict(now=now) return dict(now=now)
@@ -153,23 +154,29 @@ def index(env):
metrics['avg_resources_node'] = "{0:10.0f}".format( metrics['avg_resources_node'] = "{0:10.0f}".format(
avg_resources_node['Value']) avg_resources_node['Value'])
else: else:
conditions = ", ".join('["=", "{0}", "{1}"]'.format(field, env) query = AndOperator()
for field in ['catalog_environment', 'facts_environment']) query.add(EqualsOperator('catalog_environment', env))
query.add(EqualsOperator('facts_environment', env))
num_nodes_query = ExtractOperator()
num_nodes_query.add_field(FunctionOperator('count'))
num_nodes_query.add_query(query)
if app.config['OVERVIEW_FILTER'] != None: if app.config['OVERVIEW_FILTER'] != None:
conditions += ", {0}".format(app.config['OVERVIEW_FILTER']) query.add(app.config['OVERVIEW_FILTER'])
query = '["and", {0}]'.format(conditions)
num_resources_query = ExtractOperator()
num_resources_query.add_field(FunctionOperator('count'))
num_resources_query.add_query(EqualsOperator("environment", env))
num_nodes = get_or_abort( num_nodes = get_or_abort(
puppetdb._query, puppetdb._query,
'nodes', 'nodes',
query='["extract", [["function", "count"]],["and", {0}]]'.format( query=num_nodes_query)
",".join('["=", "{0}", "{1}"]'.format(field, env)
for field in ['catalog_environment', 'facts_environment'])))
num_resources = get_or_abort( num_resources = get_or_abort(
puppetdb._query, puppetdb._query,
'resources', 'resources',
query='["extract", [["function", "count"]],' \ query=num_resources_query)
'["=", "environment", "{0}"]]'.format(
env))
metrics['num_nodes'] = num_nodes[0]['count'] metrics['num_nodes'] = num_nodes[0]['count']
metrics['num_resources'] = num_resources[0]['count'] metrics['num_resources'] = num_resources[0]['count']
try: try:
@@ -238,9 +245,9 @@ def nodes(env):
if env == '*': if env == '*':
query = None query = None
else: else:
query = '["and", {0}]'.format( query = AndOperator()
", ".join('["=", "{0}", "{1}"]'.format(field, env) query.add(EqualsOperator("catalog_environment", env))
for field in ['catalog_environment', 'facts_environment'])), query.add(EqualsOperator("facts_environment", env))
status_arg = request.args.get('status', '') status_arg = request.args.get('status', '')
nodelist = puppetdb.nodes( nodelist = puppetdb.nodes(
@@ -304,15 +311,14 @@ def inventory(env):
fact_desc.append(description) fact_desc.append(description)
fact_names.append(name) fact_names.append(name)
if env == '*': query = AndOperator()
query = '["or", {0}]]'.format( fact_query = OrOperator()
', '.join('["=", "name", "{0}"]'.format(name) fact_query.add([EqualsOperator("name", name) for name in fact_names])
for name in fact_names))
else: if env != '*':
query = '["and", ["=", "environment", "{0}"], ["or", {1}]]'.format( query.add(EqualsOperator("environment", env))
env,
', '.join('["=", "name", "{0}"]'.format(name) query.add(fact_query)
for name in fact_names))
# get all the facts from PuppetDB # get all the facts from PuppetDB
facts = puppetdb.facts(query=query) facts = puppetdb.facts(query=query)
@@ -351,12 +357,12 @@ def node(env, node_name):
""" """
envs = environments() envs = environments()
check_env(env, envs) check_env(env, envs)
query = AndOperator()
if env == '*': if env != '*':
query = '["=", "certname", "{0}"]]'.format(node_name) query.add(EqualsOperator("environment", env))
else:
query='["and", ["=", "environment", "{0}"],' \ query.add(EqualsOperator("certname", node_name))
'["=", "certname", "{1}"]]'.format(env, node_name),
node = get_or_abort(puppetdb.node, node_name) node = get_or_abort(puppetdb.node, node_name)
facts = node.facts() facts = node.facts()
@@ -418,14 +424,14 @@ def reports(env, page):
envs = environments() envs = environments()
check_env(env, envs) check_env(env, envs)
limit = request.args.get('limit', app.config['REPORTS_COUNT']) limit = request.args.get('limit', app.config['REPORTS_COUNT'])
reports_query = None
total_query = ExtractOperator()
if env == '*': total_query.add_field(FunctionOperator("count"))
reports_query = None
total_query = '["extract", [["function", "count"]], ["~", "certname", ""]]' if env != '*':
else: reports_query = EqualsOperator("environment", env)
reports_query = '["=", "environment", "{0}"]'.format(env) total_query.add_query(reports_query)
total_query = '["extract", [["function", "count"]],'\
'["and", ["=", "environment", "{0}"]]]'.format(env)
try: try:
paging_args = {'limit': int(limit)} paging_args = {'limit': int(limit)}
@@ -499,19 +505,16 @@ def reports_node(env, node_name, page):
""" """
envs = environments() envs = environments()
check_env(env, envs) check_env(env, envs)
query = AndOperator()
total_query = ExtractOperator()
if env == '*': total_query.add_field(FunctionOperator("count"))
query = '["=", "certname", "{0}"]'.format(node_name)
total_query = '["extract", [["function", "count"]],'\ if env != '*':
'["=", "certname", "{0}"]'.format(node_name) query.add(EqualsOperator("environment", env))
else:
query='["and",' \ query.add(EqualsOperator("certname", node_name))
'["=", "environment", "{0}"],' \ total_query.add_query(query)
'["=", "certname", "{1}"]]'.format(env, node_name)
total_query = '["extract", [["function", "count"]],' \
'["and",' \
'["=", "environment", "{0}"],' \
'["=", "certname", "{1}"]]]'.format(env, node_name)
reports = get_or_abort(puppetdb.reports, reports = get_or_abort(puppetdb.reports,
query=query, query=query,
@@ -582,15 +585,18 @@ def report(env, node_name, report_id):
""" """
envs = environments() envs = environments()
check_env(env, envs) check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
if env == '*':
query = '["and", ["=", "certname", "{0}"],' \
'["or", ["=", "hash", "{1}"], ["=", "configuration_version", "{1}"]]]'.format(
node_name, report_id)
else:
query = '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"],' \
'["or", ["=", "hash", "{2}"], ["=", "configuration_version", "{2}"]]]'.format(
env, node_name, report_id)
reports = puppetdb.reports(query=query) reports = puppetdb.reports(query=query)
try: try:
@@ -656,10 +662,12 @@ def fact(env, fact):
render_graph = False render_graph = False
if fact in graph_facts: if fact in graph_facts:
render_graph = True render_graph = True
if env == '*': if env == '*':
query = None query = None
else: else:
query = '["=", "environment", "{0}"]'.format(env) query = EqualsOperator("environment", env)
localfacts = [f for f in yield_or_stop(puppetdb.facts( localfacts = [f for f in yield_or_stop(puppetdb.facts(
name=fact, query=query))] name=fact, query=query))]
return Response(stream_with_context(stream_template( return Response(stream_with_context(stream_template(
@@ -689,7 +697,8 @@ def fact_value(env, fact, value):
if env == '*': if env == '*':
query = None query = None
else: else:
query = '["=", "environment", "{0}"]'.format(env) query = EqualsOperator("environment", env)
facts = get_or_abort(puppetdb.facts, facts = get_or_abort(puppetdb.facts,
name=fact, name=fact,
value=value, value=value,
@@ -803,12 +812,13 @@ def catalogs(env):
if app.config['ENABLE_CATALOG']: if app.config['ENABLE_CATALOG']:
nodenames = [] nodenames = []
catalog_list = [] catalog_list = []
if env == '*': query = AndOperator()
query = '["null?", "catalog_timestamp", false]]'
else: if env != '*':
query = '["and",' \ query.add(EqualsOperator("catalog_environment", env))
'["=", "catalog_environment", "{0}"],' \
'["null?", "catalog_timestamp", false]]'.format(env), query.add(NullOperator("catalog_timestamp", False))
nodes = get_or_abort(puppetdb.nodes, nodes = get_or_abort(puppetdb.nodes,
query=query, query=query,
with_status=False, with_status=False,
@@ -942,14 +952,18 @@ def radiator(env):
'puppetlabs.puppetdb.population:name=num-nodes') 'puppetlabs.puppetdb.population:name=num-nodes')
num_nodes = metrics['Value'] num_nodes = metrics['Value']
else: else:
query = '["and", {0}]]'.format( query = AndOperator()
",".join('["=", "{0}", "{1}"]'.format(field, env) metric_query = ExtractOperator()
for field in ['catalog_environment', 'facts_environment']))
query.add(EqualsOperator("catalog_environment", env))
query.add(EqualsOperator("facts_environment", env))
metric_query.add_field(FunctionOperator('count'))
metric_query.add_query(query)
metrics = get_or_abort( metrics = get_or_abort(
puppetdb._query, puppetdb._query,
'nodes', 'nodes',
query='["extract", [["function", "count"]],{0}'.format( query=metric_query)
query))
num_nodes = metrics[0]['count'] num_nodes = metrics[0]['count']

2
setup.py
View File

@@ -32,7 +32,7 @@ setup(
"Flask >= 0.10.1", "Flask >= 0.10.1",
"Flask-WTF >= 0.9.4, <= 0.9.5", "Flask-WTF >= 0.9.4, <= 0.9.5",
"WTForms < 2.0", "WTForms < 2.0",
"pypuppetdb >= 0.2.1, < 0.3.0", "pypuppetdb >= 0.3.0, < 0.4.0",
], ],
keywords="puppet puppetdb puppetboard", keywords="puppet puppetdb puppetboard",
classifiers=[ classifiers=[