Merge pull request #353 from redref/catalogs

Revamp catalog page with paging

puppetboard/app.py

@@ -42,6 +42,12 @@ REPORTS_COLUMNS = [
      'name': 'Agent version'},
 ]
 
+CATALOGS_COLUMNS = [
+    {'attr': 'certname', 'name': 'Certname', 'type': 'node'},
+    {'attr': 'catalog_timestamp', 'name': 'Compile Time'},
+    {'attr': 'form', 'name': 'Compare'},
+]
+
 app = Flask(__name__)
 
 app.config.from_object('puppetboard.default_settings')
@@ -835,9 +841,14 @@ def metric(env, metric):
         current_env=env)
 
 
-@app.route('/catalogs', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
+@app.route('/catalogs',
-@app.route('/<env>/catalogs')
+           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
-def catalogs(env):
+                     'compare': None})
+@app.route('/<env>/catalogs', defaults={'compare': None})
+@app.route('/catalogs/compare/<compare>',
+           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
+@app.route('/<env>/catalogs/compare/<compare>')
+def catalogs(env, compare):
     """Lists all nodes with a compiled catalog.
 
     :param env: Find the nodes with this catalog_environment value
@@ -846,53 +857,80 @@ def catalogs(env):
     envs = environments()
     check_env(env, envs)
 
-    if app.config['ENABLE_CATALOG']:
+    if not app.config['ENABLE_CATALOG']:
-        nodenames = []
-        catalog_list = []
-        query = AndOperator()
-
-        if env != '*':
-            query.add(EqualsOperator("catalog_environment", env))
-
-        query.add(NullOperator("catalog_timestamp", False))
-
-        order_by_str = '[{"field": "certname", "order": "asc"}]'
-        nodes = get_or_abort(puppetdb.nodes,
-                             query=query,
-                             with_status=False,
-                             order_by=order_by_str)
-        nodes, temp = tee(nodes)
-
-        for node in temp:
-            nodenames.append(node.name)
-
-        for node in nodes:
-            table_row = {
-                'name': node.name,
-                'catalog_timestamp': node.catalog_timestamp
-            }
-
-            if len(nodenames) > 1:
-                form = CatalogForm()
-
-                form.compare.data = node.name
-                form.against.choices = [(x, x) for x in nodenames
-                                        if x != node.name]
-                table_row['form'] = form
-            else:
-                table_row['form'] = None
-
-            catalog_list.append(table_row)
-
-        return render_template(
-            'catalogs.html',
-            nodes=catalog_list,
-            envs=envs,
-            current_env=env)
-    else:
         log.warn('Access to catalog interface disabled by administrator')
         abort(403)
 
+    return render_template(
+        'catalogs.html',
+        compare=compare,
+        columns=CATALOGS_COLUMNS,
+        envs=envs,
+        current_env=env)
+
+
+@app.route('/catalogs/json',
+           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
+                     'compare': None})
+@app.route('/<env>/catalogs/json', defaults={'compare': None})
+@app.route('/catalogs/compare/<compare>/json',
+           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
+@app.route('/<env>/catalogs/compare/<compare>/json')
+def catalogs_ajax(env, compare):
+    """Server data to catalogs as JSON to Jquery datatables
+    """
+    draw = int(request.args.get('draw', 0))
+    start = int(request.args.get('start', 0))
+    length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
+    paging_args = {'limit': length, 'offset': start}
+    search_arg = request.args.get('search[value]')
+    order_column = int(request.args.get('order[0][column]', 0))
+    order_filter = CATALOGS_COLUMNS[order_column].get(
+        'filter', CATALOGS_COLUMNS[order_column]['attr'])
+    order_dir = request.args.get('order[0][dir]', 'asc')
+    order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
+
+    envs = environments()
+    check_env(env, envs)
+
+    query = AndOperator()
+    if env != '*':
+        query.add(EqualsOperator("catalog_environment", env))
+    if search_arg:
+        query.add(RegexOperator("certname", r"%s" % search_arg))
+    query.add(NullOperator("catalog_timestamp", False))
+
+    nodes = get_or_abort(puppetdb.nodes,
+                         query=query,
+                         include_total=True,
+                         order_by=order_args,
+                         **paging_args)
+
+    catalog_list = []
+    total = None
+    for node in nodes:
+        if total is None:
+            total = puppetdb.total
+
+        catalog_list.append({
+            'certname': node.name,
+            'catalog_timestamp': node.catalog_timestamp,
+            'form': compare,
+        })
+
+    if total is None:
+        total = 0
+
+    return render_template(
+        'catalogs.json.tpl',
+        total=total,
+        total_filtered=total,
+        draw=draw,
+        columns=CATALOGS_COLUMNS,
+        catalogs=catalog_list,
+        envs=envs,
+        current_env=env)
+
 
 @app.route('/catalog/<node_name>',
            defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
@@ -918,40 +956,6 @@ def catalog_node(env, node_name):
         abort(403)
 
 
-@app.route('/catalog/submit', methods=['POST'],
-           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
-@app.route('/<env>/catalog/submit', methods=['POST'])
-def catalog_submit(env):
-    """Receives the submitted form data from the catalogs page and directs
-       the users to the comparison page. Directs users back to the catalogs
-       page if no form submission data is found.
-
-    :param env: This parameter only directs the response page to the right
-       environment. If this environment does not exist return the use to the
-       catalogs page with the right environment.
-    :type env: :obj:`string`
-    """
-    envs = environments()
-    check_env(env, envs)
-
-    if app.config['ENABLE_CATALOG']:
-        form = CatalogForm(request.form)
-
-        form.against.choices = [(form.against.data, form.against.data)]
-        if form.validate_on_submit():
-            compare = form.compare.data
-            against = form.against.data
-            return redirect(
-                url_for('catalog_compare',
-                        env=env,
-                        compare=compare,
-                        against=against))
-        return redirect(url_for('catalogs', env=env))
-    else:
-        log.warn('Access to catalog interface disabled by administrator')
-        abort(403)
-
-
 @app.route('/catalogs/compare/<compare>...<against>',
            defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
 @app.route('/<env>/catalogs/compare/<compare>...<against>')

puppetboard/templates/catalogs.html

@@ -1,40 +1,21 @@
 {% extends 'layout.html' %}
 {% import '_macros.html' as macros %}
 {% block content %}
-<div class="ui fluid icon input hide" style="margin-bottom:20px">
+<table id="catalogs_table" class='ui very basic table stackable'>
-  <input autofocus="autofocus" class="filter-table" placeholder="Type here to filter...">
-</div>
-<table class='ui very basic very compact table nodes'>
   <thead>
     <tr>
-      <th></th>
+      {% for column in columns %}
-      <th>Certname</th>
+      <th>{{ column.name }}</th>
-      <th>Compile Time</th>
+      {% endfor %}
-      <th>Compare With</th>
     </tr>
   </thead>
-  <tbody class="searchable">
+  <tbody>
-    {% for node in nodes %}
-      <tr>
-          <td></td>
-          <td><a href="{{url_for('node', env=current_env, node_name=node.name)}}">{{node.name}}</a></td>
-          <td><a rel="utctimestamp" href="{{url_for('catalog_node', env=current_env, node_name=node.name)}}">{{node.catalog_timestamp}}</a></td>
-          <td>
-            {% if node.form %}
-              <div class="ui action input">
-                <form method="POST" action="{{url_for('catalog_submit', env=current_env)}}">
-                  {{node.form.csrf_token}}
-                  <div class="field inline">
-                    {{node.form.compare}}
-                    {{node.form.against}}
-                    <input type="submit" class="ui submit button" style="height:auto;" value="Compare"/>
-                  </div>
-                </form>
-              </div>
-            {% endif %}
-          </td>
-      </tr>
-    {% endfor %}
   </tbody>
 </table>
 {% endblock content %}
+{% block onload_script %}
+{% macro extra_options(caller) %}
+  "order": [[ 0, "asc" ]],
+{% endmacro %}
+{{ macros.datatable_init(table_html_id="catalogs_table", ajax_url=url_for('catalogs_ajax', env=current_env, compare=compare), default_length=config.NORMAL_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }}
+{% endblock onload_script %}

puppetboard/templates/catalogs.json.tpl

@@ -0,0 +1,40 @@
+{
+  "draw": {{draw}},
+  "recordsTotal": {{total}},
+  "recordsFiltered": {{total_filtered}},
+  "data": [
+    {% for catalog in catalogs -%}
+      {%- if not loop.first %},{%- endif -%}
+      [
+        {%- for column in columns -%}
+          {%- if not loop.first %},{%- endif -%}
+          {%- if column.attr == 'catalog_timestamp' -%}
+            "<a rel=\"utctimestamp\" href=\"{{url_for('catalog_node', env=current_env, node_name=catalog.certname)}}\">{{ catalog.catalog_timestamp }}</a>"
+          {%- elif column.type == 'node' -%}
+            {% filter jsonprint %}<a href="{{url_for('node', env=current_env, node_name=catalog.certname)}}">{{ catalog.certname }}</a>{% endfilter %}
+          {%- elif column.attr == 'form' -%}
+            {% filter jsonprint -%}
+              <div class="ui action input">
+                {%- if catalog.form -%}
+                <form method="GET" action="{{url_for('catalog_compare', env=current_env, compare=catalog.form, against=catalog.certname)}}">
+                {%- else -%}
+                <form method="GET" action="{{url_for('catalogs', env=current_env, compare=catalog.certname)}}">
+                {%- endif -%}
+                  <div class="field inline">
+                    {%- if catalog.form -%}
+                      <input type="submit" class="ui submit button" style="height:auto;" value="Compare with {{ catalog.form }}"/>
+                    {%- else -%}
+                      <input type="submit" class="ui submit button" style="height:auto;" value="Compare with ..."/>
+                    {%- endif -%}
+                  </div>
+                </form>
+              </div>
+            {%- endfilter -%}
+          {%- else -%}
+            ""
+          {%- endif -%}
+        {%- endfor -%}
+      ]
+    {% endfor %}
+  ]
+}

test/test_app.py

@@ -71,13 +71,6 @@ def mock_puppetdb_default_nodes(mocker):
              catalog_timestamp='2013-08-01T09:57:00.000Z',
              facts_timestamp='2013-08-01T09:57:00.000Z',
              status_report='unchanged'),
-        Node('_', 'node-skipped',
-             report_timestamp='2013-08-01T09:57:00.000Z',
-             latest_report_hash='1234567',
-             catalog_timestamp='2013-08-01T09:57:00.000Z',
-             facts_timestamp='2013-08-01T09:57:00.000Z',
-             status_report='skipped')
-
     ]
     return mocker.patch.object(app.puppetdb, 'nodes',
                                return_value=iter(node_list))
@@ -443,7 +436,6 @@ def test_radiator_view_json(client, mocker,
     assert json_data['noop'] == 1
     assert json_data['failed'] == 1
     assert json_data['changed'] == 1
-    assert json_data['skipped'] == 1
     assert json_data['unchanged'] == 1
 
 
@@ -537,9 +529,6 @@ def test_json_daily_reports_chart_ok(client, mocker):
         ]
     }
 
-    import logging
-    logging.error(query_data)
-
     dbquery = MockDbQuery(query_data)
 
     mocker.patch.object(app.puppetdb, '_query', side_effect=dbquery.get)
@@ -558,3 +547,76 @@ def test_json_daily_reports_chart_ok(client, mocker):
         cur_day = next_day
 
     assert rv.status_code == 200
+
+
+def test_catalogs_disabled(client, mocker,
+                           mock_puppetdb_environments,
+                           mock_puppetdb_default_nodes):
+    app.app.config['ENABLE_CATALOG'] = False
+    rv = client.get('/catalogs')
+    assert rv.status_code == 403
+
+
+def test_catalogs_view(client, mocker,
+                       mock_puppetdb_environments,
+                       mock_puppetdb_default_nodes):
+    app.app.config['ENABLE_CATALOG'] = True
+    rv = client.get('/catalogs')
+    assert rv.status_code == 200
+    soup = BeautifulSoup(rv.data, 'html.parser')
+    assert soup.title.contents[0] == 'Puppetboard'
+
+
+def test_catalogs_json(client, mocker,
+                       mock_puppetdb_environments,
+                       mock_puppetdb_default_nodes):
+    app.app.config['ENABLE_CATALOG'] = True
+    rv = client.get('/catalogs/json')
+    assert rv.status_code == 200
+
+    result_json = json.loads(rv.data.decode('utf-8'))
+    assert 'data' in result_json
+
+    for line in result_json['data']:
+        assert len(line) == 3
+        found_status = None
+        for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']:
+            val = BeautifulSoup(line[0], 'html.parser').find_all(
+                'a', {"href": "/node/node-%s/" % status})
+            if len(val) == 1:
+                found_status = status
+                break
+        assert found_status, 'Line does not match any known status'
+
+        val = BeautifulSoup(line[2], 'html.parser').find_all(
+            'form', {"method": "GET",
+                     "action": "/catalogs/compare/node-%s" % found_status})
+        assert len(val) == 1
+
+
+def test_catalogs_json_compare(client, mocker,
+                               mock_puppetdb_environments,
+                               mock_puppetdb_default_nodes):
+    app.app.config['ENABLE_CATALOG'] = True
+    rv = client.get('/catalogs/compare/node-unreported/json')
+    assert rv.status_code == 200
+
+    result_json = json.loads(rv.data.decode('utf-8'))
+    assert 'data' in result_json
+
+    for line in result_json['data']:
+        assert len(line) == 3
+        found_status = None
+        for status in ['failed', 'changed', 'unchanged', 'noop', 'unreported']:
+            val = BeautifulSoup(line[0], 'html.parser').find_all(
+                'a', {"href": "/node/node-%s/" % status})
+            if len(val) == 1:
+                found_status = status
+                break
+        assert found_status, 'Line does not match any known status'
+
+        val = BeautifulSoup(line[2], 'html.parser').find_all(
+            'form', {"method": "GET",
+                     "action": "/catalogs/compare/node-unreported...node-%s" %
+                     found_status})
+        assert len(val) == 1