Add summary and activity screens, remove private datasets from counts
Add summary and activity screens, remove private datasets from counts

file:b/README.md (new)
--- /dev/null
+++ b/README.md
@@ -1,1 +1,8 @@
+# ckanext-dga-stats
 
+Fork of CKAN's built-in Statistics plugin modified for data.gov.au
+
+* Remove private datasets from all statistics (except top users)
+* Add summary page
+* Add activity summary page
+

--- /dev/null
+++ b/ckanext/__init__.py
@@ -1,1 +1,8 @@
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)
 

 Binary files /dev/null and b/ckanext/__init__.pyc differ
--- a/ckanext/dga-stats/__init__.py
+++ /dev/null
@@ -1,2 +1,1 @@
-# empty file needed for pylons to find templates in this directory
 

 Binary files a/ckanext/dga-stats/__init__.pyc and /dev/null differ
--- a/ckanext/dga-stats/controller.py
+++ /dev/null
@@ -1,51 +1,1 @@
-import ckan.plugins as p
-from ckan.lib.base import BaseController, config
-import stats as stats_lib
-import ckan.lib.helpers as h
 
-class StatsController(BaseController):
-
-    def index(self):
-        c = p.toolkit.c
-        stats = stats_lib.Stats()
-        rev_stats = stats_lib.RevisionStats()
-        c.top_rated_packages = stats.top_rated_packages()
-        c.most_edited_packages = stats.most_edited_packages()
-        c.largest_groups = stats.largest_groups()
-        c.top_tags = stats.top_tags()
-        c.top_package_owners = stats.top_package_owners()
-        c.new_packages_by_week = rev_stats.get_by_week('new_packages')
-        c.deleted_packages_by_week = rev_stats.get_by_week('deleted_packages')
-        c.num_packages_by_week = rev_stats.get_num_packages_by_week()
-        c.package_revisions_by_week = rev_stats.get_by_week('package_revisions')
-
-        # Used in the legacy CKAN templates.
-        c.packages_by_week = []
-
-        # Used in new CKAN templates gives more control to the templates for formatting.
-        c.raw_packages_by_week = []
-        for week_date, num_packages, cumulative_num_packages in c.num_packages_by_week:
-            c.packages_by_week.append('[new Date(%s), %s]' % (week_date.replace('-', ','), cumulative_num_packages))
-            c.raw_packages_by_week.append({'date': h.date_str_to_datetime(week_date), 'total_packages': cumulative_num_packages})
-
-        c.all_package_revisions = []
-        c.raw_all_package_revisions = []
-        for week_date, revs, num_revisions, cumulative_num_revisions in c.package_revisions_by_week:
-            c.all_package_revisions.append('[new Date(%s), %s]' % (week_date.replace('-', ','), num_revisions))
-            c.raw_all_package_revisions.append({'date': h.date_str_to_datetime(week_date), 'total_revisions': num_revisions})
-
-        c.new_datasets = []
-        c.raw_new_datasets = []
-        for week_date, pkgs, num_packages, cumulative_num_packages in c.new_packages_by_week:
-            c.new_datasets.append('[new Date(%s), %s]' % (week_date.replace('-', ','), num_packages))
-            c.raw_new_datasets.append({'date': h.date_str_to_datetime(week_date), 'new_packages': num_packages})
-
-        return p.toolkit.render('ckanext/stats/index.html')
-
-    def leaderboard(self, id=None):
-        c = p.toolkit.c
-        c.solr_core_url = config.get('ckanext.stats.solr_core_url',
-                'http://solr.okfn.org/solr/ckan')
-        return p.toolkit.render('ckanext/stats/leaderboard.html')
-
-

--- a/ckanext/dga-stats/plugin.py
+++ /dev/null
@@ -1,28 +1,1 @@
-from logging import getLogger
 
-import ckan.plugins as p
-
-log = getLogger(__name__)
-
-class StatsPlugin(p.SingletonPlugin):
-    '''Stats plugin.'''
-
-    p.implements(p.IRoutes, inherit=True)
-    p.implements(p.IConfigurer, inherit=True)
-
-    def after_map(self, map):
-        map.connect('stats', '/stats',
-            controller='ckanext.stats.controller:StatsController',
-            action='index')
-        map.connect('stats_action', '/stats/{action}',
-            controller='ckanext.stats.controller:StatsController')
-        return map
-
-    def update_config(self, config):
-        templates = 'templates'
-        if p.toolkit.asbool(config.get('ckan.legacy_templates', False)):
-                templates = 'templates_legacy'
-        p.toolkit.add_template_directory(config, templates)
-        p.toolkit.add_public_directory(config, 'public')
-        p.toolkit.add_resource('public/ckanext/stats', 'ckanext_stats')
-

 Binary files a/ckanext/dga-stats/plugin.pyc and /dev/null differ
--- a/ckanext/dga-stats/public/.gitignore
+++ /dev/null
@@ -1,3 +1,1 @@
-**.min.js
-**.min.css
 

--- a/ckanext/dga-stats/public/__init__.py
+++ /dev/null
@@ -1,8 +1,1 @@
-# this is a namespace package
-try:
-    import pkg_resources
-    pkg_resources.declare_namespace(__name__)
-except ImportError:
-    import pkgutil
-    __path__ = pkgutil.extend_path(__path__, __name__)
 

--- a/ckanext/dga-stats/public/ckanext/__init__.py
+++ /dev/null
@@ -1,8 +1,1 @@
-# this is a namespace package
-try:
-    import pkg_resources
-    pkg_resources.declare_namespace(__name__)
-except ImportError:
-    import pkgutil
-    __path__ = pkgutil.extend_path(__path__, __name__)
 

--- a/ckanext/dga-stats/public/ckanext/stats/__init__.py
+++ /dev/null
@@ -1,8 +1,1 @@
-# this is a namespace package
-try:
-    import pkg_resources
-    pkg_resources.declare_namespace(__name__)
-except ImportError:
-    import pkgutil
-    __path__ = pkgutil.extend_path(__path__, __name__)
 

--- a/ckanext/dga-stats/public/ckanext/stats/app.js
+++ /dev/null
@@ -1,60 +1,1 @@
-jQuery(document).ready(function($) {
-	$('form').submit(function(e) {
-		e.preventDefault();
-		attribute = $('#form-attribute').val();
-		loadSolr(attribute);
-	})
-	// default! (also in html)
-	loadSolr('tags');
 
-	function loadSolr(attribute) {
-		var url = solrCoreUrl + '/select?indent=on&wt=json&facet=true&rows=0&indent=true&facet.mincount=1&facet.limit=30&q=*:*&facet.field=' + attribute;
-		function handleSolr(data) {
-			var results = [];
-			ourdata = data.facet_counts.facet_fields[attribute];
-			var newrow = {};
-			for (ii in ourdata) {
-				if (ii % 2 == 0) {
-					newrow.name = ourdata[ii];
-					if (!newrow.name) {
-						newrow.name = '[Not Specified]';
-					}
-				} else {
-					newrow.count = ourdata[ii];
-					results.push(newrow);
-					newrow = {};
-				}
-			}
-			display(results);
-		}
-
-		$.ajax({
-			url: url,
-			success: handleSolr,
-			dataType: 'jsonp',
-			jsonp: 'json.wrf'
-		});
-	}
-
-	function display(results) {
-		var list = $('#category-counts');
-		list.html('');
-		if (results.length == 0) {
-			return
-		}
-		var maximum = results[0]['count'];
-		for(ii in results) {
-			maximum = Math.max(maximum, results[ii]['count']);
-		}
-
-		$.each(results, function(idx, row) {
-			var newentry = $('<li></li>');
-			newentry.append($('<a href="#">' + row['name'] + '</a>'));
-			newentry.append($('<span class="count">' + row['count'] + '</a>'));
-			var percent = 100 * row['count'] / maximum;
-			newentry.append($('<span class="index" style="width: ' + percent + '%"></span>'));
-			list.append(newentry);
-		});
-	}
-});
-

--- a/ckanext/dga-stats/public/ckanext/stats/css/stats.css
+++ /dev/null
@@ -1,17 +1,1 @@
-.tab-content h2 {
-  margin-bottom: 12px;
-}
 
-.js .tab-content {
-  padding-top: 20px;
-  padding-bottom: 20px;
-  margin-top: 0;
-}
-
-.module-plot-canvas {
-  display: block;
-  width: 650px;
-  height: 300px;
-  margin: 20px 0;
-}
-

--- a/ckanext/dga-stats/public/ckanext/stats/demo.html
+++ /dev/null
@@ -1,26 +1,1 @@
-<html>
-  <head>
-    <script type="text/javascript">
-      var solrCoreUrl = 'http://solr.okfn.org/solr/ckan';
-    </script>
-    <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js"></script>
-    <script type="text/javascript" src="app.js"></script>
 
-    <link type="text/css" rel="stylesheet" media="all" href="style.css" />
-  </head>
-  <body>
-    <h1>CKAN Dataset Leaderboard</h1>
-    <p>Choose a dataset attribute and find out which categories in that area have the most datasets. E.g. tags, groups, license, res_format, country.</p>
-    <form>
-      <label for="category">Choose area</label>
-      <input type="text" value="tags" name="attribute" id="form-attribute" />
-      <input type="submit" value="Dataset Counts &raquo;" name="submit" />
-    </form>
-
-    <div class="category-counts">
-      <ul class="chartlist" id="category-counts">
-      </ul>
-    </div>
-  </body>
-</html>
-

--- a/ckanext/dga-stats/public/ckanext/stats/javascript/modules/plot.js
+++ /dev/null
@@ -1,210 +1,1 @@
-/* A quick module for generating flot charts from an HTML table. Options can
- * be passed directly to flot using the data-module-* attributes. The tables
- * are currently expected to be marked up as follows:
- *
- *   <table data-module="plot">
- *     <thead>
- *       <tr>
- *         <th>X Axis</th>
- *         <th>Series A Legend</th>
- *         <th>Series B Legend</th>
- *       </tr>
- *     </thead>
- *     <tbody>
- *       <tr>
- *         <th>X Value</th>
- *         <td>Series A Y Value</td>
- *         <td>Series B Y Value</td>
- *       </tr>
- *       ...
- *     </tbody>
- *   </table>
- *
- * Points are pulled out of the th/td elements using innerHTML or by looking
- * for a data-value attribute. This is useful when a more readable value
- * needs to be used in the elements contents (eg. dates). A data-type attribute
- * can also be applied to parse the value. Only data-type="date" is currently
- * supported and expects data-value to be a unix timestamp.
- */
-this.ckan.module('plot', function (jQuery, _) {
-  return {
-    /* Holds the jQuery.plot() object when created */
-    graph: null,
 
-    /* Holds the canvas container when created */
-    canvas: null,
-
-    /* Default options */
-    options: {
-      xaxis: {},
-      yaxis: {},
-      legend: {position: 'nw'},
-      colors: ['#ffcc33', '#ff8844']
-    },
-
-    /* Sets up the canvas element and parses the table.
-     *
-     * Returns nothing.
-     */
-    initialize: function () {
-      jQuery.proxyAll(this, /_on/);
-
-      if (!this.el.is('table')) {
-        throw new Error('CKAN module plot can only be called on table elements');
-      }
-
-      this.setupCanvas();
-
-      // Because the canvas doesn't render correctly unless visible we must
-      // listen for events that reveal the canvas and then try and re-render.
-      // Currently the most common of these is the "shown" event triggered by
-      // the tabs plugin.
-      this.sandbox.body.on("shown", this._onShown);
-      this.data = this.parseTable(this.el);
-
-      this.draw();
-    },
-
-    /* Removes event handlers when the module is removed from the DOM.
-     *
-     * Returns nothing.
-     */
-    teardown: function () {
-      this.sandbox.body.off("shown", this._onShown);
-    },
-
-    /* Creates the canvas wrapper and removes the table from the document.
-     *
-     * Returns nothing.
-     */
-    setupCanvas: function () {
-      this.canvas = jQuery('<div class="module-plot-canvas">');
-      this.el.replaceWith(this.canvas);
-    },
-
-    /* Attempts to draw the chart if the canvas is visible. If not visible the
-     * graph does not render correctly. So we keep trying until it is.
-     *
-     * Examples
-     *
-     *   module.draw();
-     *
-     * Returns nothing.
-     */
-    draw: function () {
-      if (!this.drawn && this.canvas.is(':visible')) {
-        this.graph = jQuery.plot(this.canvas, this.data, this.options);
-      }
-    },
-
-    /* Parses an HTML table element to build the data array for the chart.
-     * The thead element provides the axis and labels for the series. The
-     * first column in the tbody is used for the x-axis and subsequent
-     * columns are the series.
-     *
-     * table - A table element to parse.
-     *
-     * Examples
-     *
-     *   module.parseTable(module.el);
-     *
-     * Returns data object suitable for use in jQuery.plot().
-     */
-    parseTable: function (table) {
-      var data = [];
-      var _this = this;
-
-      var headings = table.find('thead tr:first th').map(function () {
-        return this.innerHTML;
-      });
-
-      table.find('tbody tr').each(function (row) {
-        var element = jQuery(this);
-        var x = [];
-
-        x[row] = _this.getValue(element.find('th'));
-
-        element.find('td').each(function (series) {
-          var value   = _this.getValue(this);
-          var label = headings[series + 1];
-
-          data[series] = data[series] || {data: [], label: label};
-          data[series].data[row] = [x[row], value];
-        });
-      });
-
-      return data;
-    },
-
-    /* Retrieves the value from a td/th element. This first looks for a
-     * data-value attribute on the element otherwise uses the element
-     * text contents.
-     *
-     * A data-type attribute can also be provided to tell the module how
-     * to deal with the element. By default we let jQuery.data() handle
-     * the parsing but this can provide additional data. See .parseValue()
-     * for more info.
-     *
-     * cell - An element to extract a value from.
-     *
-     * Examples
-     *
-     *   var element = jQuery('<td data-value="10">Ten</td>');
-     *   module.getValue(element); //=> 10
-     *
-     *   var element = jQuery('<td>20</td>');
-     *   module.getValue(element); //=> 20
-     *
-     *   var element = jQuery('<td data-type="date">1343747094</td>');
-     *   module.getValue(element); //=> <Date Tue Jul 31 2012 16:04:54 GMT+0100 (BST)>
-     *
-     * Returns the parsed value.
-     */
-    getValue: function (cell) {
-      var item  = cell instanceof jQuery ? cell : jQuery(cell);
-      var type  = item.data('type')  || 'string';
-      var value = item.data('value') || item.text();
-      return this.parseValue(value, type);
-    },
-
-    /* Provides the ability to further format a value.
-     *
-     * If date is provided as a type then it expects value to be a unix
-     * timestamp in seconds.
-     *
-     * value - The value extracted from the element.
-     * type  - A type string, currently only supports "date".
-     *
-     * Examples
-     *
-     *   module.parseValue(10); // => 10
-     *   module.parseValue("cat"); // => "cat"
-     *   module.parseValue(1343747094, 'date'); // => <Date Tue Jul 31 2012 16:04:54 GMT+0100 (BST)>
-     *
-     * Returns the parsed value.
-     */
-    parseValue: function (value, type) {
-      if (type === 'date') {
-        value = new Date(parseInt(value, 10) * 1000);
-        if (!value) {
-          value = 0;
-        }
-      }
-      return value;
-    },
-
-    /* Event handler for when tabs are toggled. Determines if the canvas
-     * resides in the shown element and attempts to re-render.
-     *
-     * event - The shown event object.
-     *
-     * Returns nothing.
-     */
-    _onShown: function (event) {
-      if (!this.drawn && jQuery.contains(jQuery(event.target.hash)[0], this.canvas[0])) {
-        this.draw();
-      }
-    }
-  };
-});
-

--- a/ckanext/dga-stats/public/ckanext/stats/javascript/modules/stats-nav.js
+++ /dev/null
@@ -1,37 +1,1 @@
-/* Quick module to enhance the Bootstrap tags plug-in to update the url
- * hash when a tab changes to allow the user to bookmark the page.
- *
- * Each tab id must use a prefix which which will be stripped from the hash.
- * This is to prevent the page jumping when the hash fragment changes.
- *
- * prefix - The prefix used on the ids.
- *
- */
-this.ckan.module('stats-nav', {
-  /* An options object */
-  options: {
-    prefix: 'stats-'
-  },
 
-  /* Initializes the module and sets up event listeners.
-   *
-   * Returns nothing.
-   */
-  initialize: function () {
-    var location = this.sandbox.location;
-    var prefix = this.options.prefix;
-    var hash = location.hash.slice(1);
-    var selected = this.$('[href^=#' + prefix + hash + ']');
-
-    // Update the hash fragment when the tab changes.
-    this.el.on('shown', function (event) {
-      location.hash = event.target.hash.slice(prefix.length + 1);
-    });
-
-    // Show the current tab if the location provides one.
-    if (selected.length) {
-      selected.tab('show');
-    }
-  }
-});
-

--- a/ckanext/dga-stats/public/ckanext/stats/resource.config
+++ /dev/null
@@ -1,13 +1,1 @@
-[IE conditional]
 
-lte IE 8 = vendor/excanvas.js
-
-[groups]
-
-stats =
-    css/stats.css
-    vendor/excanvas.js
-    vendor/jquery.flot.js
-    javascript/modules/plot.js
-    javascript/modules/stats-nav.js
-

--- a/ckanext/dga-stats/public/ckanext/stats/style.css
+++ /dev/null
@@ -1,60 +1,1 @@
-div.category-counts {
-}
 
-div.category-counts-over-time {
-  clear: both;
-}
-
-/***************************
-  * CHART LISTS
-  **************************/
-
-.chartlist { 
-  float: left; 
-  border-top: 1px solid #EEE; 
-  width: 90%;
-  padding-left: 0;
-  margin-left: 0;
-}
-
-.chartlist li { 
-  position: relative;
-  display: block;  
-  border-bottom: 1px solid #EEE; 
-  _zoom: 1;
-}
-.chartlist li a { 
-  display: block; 
-  padding: 0.4em 4.5em 0.4em 0.5em;
-  position: relative; 
-  z-index: 2; 
-}
-.chartlist .count { 
-  display: block; 
-  position: absolute; 
-  top: 0; 
-  right: 0; 
-  margin: 0 0.3em; 
-  text-align: right; 
-  color: #999; 
-  font-weight: bold; 
-  font-size: 0.875em; 
-  line-height: 2em; 
-  z-index: 999;
-}
-.chartlist .index { 
-  display: block; 
-  position: absolute; 
-  top: 0; 
-  left: 0; 
-  height: 100%; 
-  background: #B8E4F5; 
-  text-indent: -9999px; 
-  overflow: hidden; 
-  line-height: 2em;
-}
-.chartlist li:hover { 
-  background: #EFEFEF; 
-}
-
-

--- a/ckanext/dga-stats/public/ckanext/stats/test/fixtures/table.html
+++ /dev/null
@@ -1,31 +1,1 @@
-<table data-module="plot">
-  <thead>
-    <tr>
-      <th>X Axis</th>
-      <th>Series A Legend</th>
-      <th>Series B Legend</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <!-- This is the x value for each series -->
-      <th data-type="date" data-value="1176073200">Apr 09, 2007</th>
-      <!-- This is the y value for series a -->
-      <td>20</td>
-      <!-- This is the y value for series b -->
-      <td>7</td>
-    </tr>
-    <tr>
-      <th data-type="date" data-value="1176678000">Apr 16, 2007</th>
-      <td>12</td>
-      <td>6</td>
-    </tr>
-    <tr>
-      <th data-type="date" data-value="1177282800">Apr 23, 2007</th>
-      <td>27</td>
-      <td>12</td>
-    </tr>
-  </tbody>
-</table>
 
-

--- a/ckanext/dga-stats/public/ckanext/stats/test/index.html
+++ /dev/null
@@ -1,60 +1,1 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <title>Mocha Tests</title>
-    <link rel="stylesheet" href="../../../base/test/vendor/mocha.css" />
-  </head>
-  <body>
-    <div id="mocha"></div>
-    <div id="fixture" style="position: absolute; top: -9999px; left: -9999px"></div>
 
-    <!-- Test Runner -->
-    <script src="../../../base/test/vendor/sinon.js"></script>
-    <script src="../../../base/test/vendor/mocha.js"></script>
-    <script src="../../../base/test/vendor/chai.js"></script>
-    <script>
-      mocha.setup('bdd');
-      var assert = chai.assert;
-
-      // Export sinon.assert methods onto assert.
-      sinon.assert.expose(assert, {prefix: ''});
-
-      var ckan = {ENV: 'testing'};
-    </script>
-
-    <!-- Source -->
-    <script src="../../../base/vendor/jed.js"></script>
-    <script src="../../../base/vendor/jquery.js"></script>
-    <script src="../../../base/vendor/bootstrap/js/bootstrap-transition.js"></script>
-    <script src="../../../base/vendor/bootstrap/js/bootstrap-alert.js"></script>
-    <script src="../../../base/javascript/plugins/jquery.inherit.js"></script>
-    <script src="../../../base/javascript/plugins/jquery.proxy-all.js"></script>
-    <script src="../../../base/javascript/sandbox.js"></script>
-    <script src="../../../base/javascript/module.js"></script>
-    <script src="../../../base/javascript/pubsub.js"></script>
-    <script src="../../../base/javascript/client.js"></script>
-    <script src="../../../base/javascript/notify.js"></script>
-    <script src="../../../base/javascript/i18n.js"></script>
-    <script src="../../../base/javascript/main.js"></script>
-    <script src="../javascript/modules/plot.js"></script>
-    <script src="../javascript/modules/stats-nav.js"></script>
-
-    <!-- Suite -->
-    <script src="./spec/modules/plot.spec.js"></script>
-    <script src="./spec/modules/stats-nav.spec.js"></script>
-
-    <script>
-      beforeEach(function () {
-        this.fixture = jQuery('#fixture').empty();
-      });
-
-      afterEach(function () {
-        this.fixture.empty();
-      });
-
-      mocha.run().globals(['ckan', 'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval']);
-    </script>
-  </body>
-</html>
-

--- a/ckanext/dga-stats/public/ckanext/stats/test/spec/modules/plot.spec.js
+++ /dev/null
@@ -1,137 +1,1 @@
-/*globals describe before beforeEach afterEach it assert sinon ckan jQuery */
-describe('ckan.module.PlotModule()', function () {
-  var PlotModule = ckan.module.registry['plot'];
 
-  before(function (done) {
-    var _this = this;
-
-    jQuery.get('./fixtures/table.html', function (html) {
-      _this.template = html;
-      done();
-    });
-  });
-
-  beforeEach(function () {
-    this.el = jQuery(this.template).appendTo(this.fixture);
-    this.sandbox = ckan.sandbox();
-    this.sandbox.body = this.fixture;
-    this.module = new PlotModule(this.el, {}, this.sandbox);
-  });
-
-  afterEach(function () {
-    this.module.teardown();
-  });
-
-  describe('.initialize()', function () {
-    it('should setup the canvas element', function () {
-      var target = sinon.stub(this.module, 'setupCanvas', this.module.setupCanvas);
-
-      this.module.initialize();
-      assert.called(target);
-    });
-
-    it('should draw the graph', function () {
-      var target = sinon.stub(this.module, 'draw');
-
-      this.module.initialize();
-      assert.called(target);
-    });
-
-    it('should listen for "shown" events on the body', function () {
-      var target = sinon.stub(this.sandbox.body, 'on');
-
-      this.module.initialize();
-      assert.called(target);
-      assert.calledWith(target, "shown", this.module._onShown);
-    });
-  });
-
-  describe('.teardown()', function () {
-    it('should remove "shown" listeners from the body', function () {
-      var target = sinon.stub(this.sandbox.body, 'off');
-
-      this.module.teardown();