Merge pull request #331 from mterzo/all_envs

Support for PuppetDB 3.x on all environments

puppetboard/app.py

@@ -21,7 +21,7 @@ from pypuppetdb.QueryBuilder import *
 
 from puppetboard.forms import (CatalogForm, QueryForm)
 from puppetboard.utils import (
-    get_or_abort, yield_or_stop,
+    get_or_abort, yield_or_stop, get_db_version,
     jsonprint, prettyprint
 )
 from puppetboard.dailychart import get_daily_reports_chart
@@ -173,15 +173,23 @@ def index(env):
         query = app.config['OVERVIEW_FILTER']
 
         prefix = 'puppetlabs.puppetdb.population'
+        query_type = ''
+
+        # Puppet DB version changed the query format from 3.2.0
+        # to 4.0 when querying mbeans
+        if get_db_version(puppetdb) < (4, 0, 0):
+            query_type = 'type=default,'
+
         num_nodes = get_or_abort(
             puppetdb.metric,
-            "{0}{1}".format(prefix, ':name=num-nodes'))
+            "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type))
         num_resources = get_or_abort(
             puppetdb.metric,
-            "{0}{1}".format(prefix, ':name=num-resources'))
+            "{0}{1}".format(prefix, ':%sname=num-resources' % query_type))
         avg_resources_node = get_or_abort(
             puppetdb.metric,
-            "{0}{1}".format(prefix, ':name=avg-resources-per-node'))
+            "{0}{1}".format(prefix,
+                            ':%sname=avg-resources-per-node' % query_type))
         metrics['num_nodes'] = num_nodes['Value']
         metrics['num_resources'] = num_resources['Value']
         metrics['avg_resources_node'] = "{0:10.0f}".format(
@@ -971,10 +979,13 @@ def radiator(env):
     check_env(env, envs)
 
     if env == '*':
+        query_type = ''
+        if get_db_version(puppetdb) < (4, 0, 0):
+            query_type = 'type=default,'
         query = None
         metrics = get_or_abort(
             puppetdb.metric,
-            'puppetlabs.puppetdb.population:name=num-nodes')
+            'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type)
         num_nodes = metrics['Value']
     else:
         query = AndOperator()

puppetboard/utils.py

@@ -24,6 +24,29 @@ def jsonprint(value):
     return json.dumps(value, indent=2, separators=(',', ': '))
 
 
+def get_db_version(puppetdb):
+    '''
+    Get the version of puppetdb.  Version form 3.2 query
+    interface is slightly different on mbeans
+    '''
+    ver = ()
+    try:
+        version = puppetdb.current_version()
+        (major, minor, build) = [int(x) for x in version.split('.')]
+        ver = (major, minor, build)
+        log.info("PuppetDB Version %d.%d.%d" % (major, minor, build))
+    except ValueError as e:
+        log.error("Unable to determine version from string: '%s'" % version)
+        ver = (4, 2, 0)
+    except HTTPError as e:
+        log.error(str(e))
+    except ConnectionError as e:
+        log.error(str(e))
+    except EmptyResponseError as e:
+        log.error(str(e))
+    return ver
+
+
 def formatvalue(value):
     if isinstance(value, str):
         return value

test/test_app.py

@@ -131,6 +131,58 @@ def test_index_all(client, mocker,
 
     base_str = 'puppetlabs.puppetdb.population:'
     query_data = {
+        'version': [{'version': '4.2.0'}],
+        'mbean': [
+            {
+                'validate': {
+                    'data': {'Value': '50'},
+                    'checks': {
+                        'path': '%sname=num-nodes' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': '60'},
+                    'checks': {
+                        'path': '%sname=num-resources' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': 60.3},
+                    'checks': {
+                        'path': '%sname=avg-resources-per-node' % base_str
+                    }
+                }
+            }
+        ]
+    }
+    dbquery = MockDbQuery(query_data)
+    mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
+    rv = client.get('/%2A/')
+
+    soup = BeautifulSoup(rv.data, 'html.parser')
+    assert soup.title.contents[0] == 'Puppetboard'
+    vals = soup.find_all('h1',
+                         {"class": "ui header darkblue no-margin-bottom"})
+
+    assert len(vals) == 3
+    assert vals[0].string == '50'
+    assert vals[1].string == '60'
+    assert vals[2].string == '        60'
+
+    assert rv.status_code == 200
+
+
+def test_index_all_older_puppetdb(client, mocker,
+                                  mock_puppetdb_environments,
+                                  mock_puppetdb_default_nodes):
+
+    base_str = 'puppetlabs.puppetdb.population:type=default,'
+    query_data = {
+        'version': [{'version': '3.2.0'}],
         'mbean': [
             {
                 'validate': {
@@ -269,6 +321,106 @@ def test_radiator_view(client, mocker,
     assert '10' in total.text
 
 
+def test_radiator_view_all(client, mocker,
+                           mock_puppetdb_environments,
+                           mock_puppetdb_default_nodes):
+    base_str = 'puppetlabs.puppetdb.population:'
+    query_data = {
+        'version': [{'version': '4.2.0'}],
+        'mbean': [
+            {
+                'validate': {
+                    'data': {'Value': '50'},
+                    'checks': {
+                        'path': '%sname=num-nodes' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': '60'},
+                    'checks': {
+                        'path': '%sname=num-resources' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': 60.3},
+                    'checks': {
+                        'path': '%sname=avg-resources-per-node' % base_str
+                    }
+                }
+            }
+        ]
+    }
+
+    dbquery = MockDbQuery(query_data)
+
+    mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
+
+    rv = client.get('/%2A/radiator')
+
+    assert rv.status_code == 200
+
+    soup = BeautifulSoup(rv.data, 'html.parser')
+    assert soup.title.contents[0] == 'Puppetboard'
+    assert soup.h1 != 'Not Found'
+    total = soup.find(class_='total')
+
+    assert '50' in total.text
+
+
+def test_radiator_view_all_old_version(client, mocker,
+                                       mock_puppetdb_environments,
+                                       mock_puppetdb_default_nodes):
+    base_str = 'puppetlabs.puppetdb.population:type=default,'
+    query_data = {
+        'version': [{'version': '3.2.0'}],
+        'mbean': [
+            {
+                'validate': {
+                    'data': {'Value': '50'},
+                    'checks': {
+                        'path': '%sname=num-nodes' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': '60'},
+                    'checks': {
+                        'path': '%sname=num-resources' % base_str
+                    }
+                }
+            },
+            {
+                'validate': {
+                    'data': {'Value': 60.3},
+                    'checks': {
+                        'path': '%sname=avg-resources-per-node' % base_str
+                    }
+                }
+            }
+        ]
+    }
+
+    dbquery = MockDbQuery(query_data)
+
+    mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
+
+    rv = client.get('/%2A/radiator')
+
+    assert rv.status_code == 200
+
+    soup = BeautifulSoup(rv.data, 'html.parser')
+    assert soup.title.contents[0] == 'Puppetboard'
+    assert soup.h1 != 'Not Found'
+    total = soup.find(class_='total')
+
+    assert '50' in total.text
+
+
 def test_radiator_view_json(client, mocker,
                             mock_puppetdb_environments,
                             mock_puppetdb_default_nodes):

test/test_utils.py

@@ -68,6 +68,18 @@ def mock_log(mocker):
     return mocker.patch('logging.log')
 
 
+@pytest.fixture
+def mock_info_log(mocker):
+    logger = logging.getLogger('puppetboard.utils')
+    return mocker.patch.object(logger, 'info')
+
+
+@pytest.fixture
+def mock_err_log(mocker):
+    logger = logging.getLogger('puppetboard.utils')
+    return mocker.patch.object(logger, 'error')
+
+
 def test_http_error(mock_log):
     err = "NotFound"
 
@@ -109,6 +121,73 @@ def test_http_empty(mock_log, mocker):
         flask_abort.assert_called_with('204')
 
 
+def test_db_version_good(mocker, mock_info_log):
+    mocker.patch.object(app.puppetdb, 'current_version', return_value='4.2.0')
+    err = 'PuppetDB Version %d.%d.%d' % (4, 2, 0)
+    result = utils.get_db_version(app.puppetdb)
+    mock_info_log.assert_called_with(err)
+    assert (4, 0, 0) < result
+    assert (4, 2, 0) == result
+    assert (3, 2, 0) < result
+    assert (4, 3, 0) > result
+    assert (5, 0, 0) > result
+    assert (4, 2, 1) > result
+
+
+def test_db_invalid_version(mocker, mock_err_log):
+    mocker.patch.object(app.puppetdb, 'current_version', return_value='4')
+    err = u"Unable to determine version from string: '%s'" % (4)
+    result = utils.get_db_version(app.puppetdb)
+    mock_err_log.assert_called_with(err)
+    assert (4, 0, 0) < result
+    assert (4, 2, 0) == result
+
+
+def test_db_http_error(mocker, mock_err_log):
+    err = "NotFound"
+
+    def raise_http_error():
+        x = Response()
+        x.status_code = 404
+        x.reason = err
+        raise HTTPError(err, response=x)
+
+    mocker.patch.object(app.puppetdb, 'current_version',
+                        side_effect=raise_http_error)
+    result = utils.get_db_version(app.puppetdb)
+    mock_err_log.assert_called_with(err)
+    assert result == ()
+
+
+def test_db_connection_error(mocker, mock_err_log):
+    err = "ConnectionError"
+
+    def connection_error():
+        x = Response()
+        x.status_code = 500
+        x.reason = err
+        raise ConnectionError(err, response=x)
+
+    mocker.patch.object(app.puppetdb, 'current_version',
+                        side_effect=connection_error)
+    result = utils.get_db_version(app.puppetdb)
+    mock_err_log.assert_called_with(err)
+    assert result == ()
+
+
+def test_db_empty_response(mocker, mock_err_log):
+    err = "Empty Response"
+
+    def connection_error():
+        raise EmptyResponseError(err)
+
+    mocker.patch.object(app.puppetdb, 'current_version',
+                        side_effect=connection_error)
+    result = utils.get_db_version(app.puppetdb)
+    mock_err_log.assert_called_with(err)
+    assert result == ()
+
+
 def test_iter():
     test_list = (0, 1, 2, 3)