Moved minor nav into util file and implemented
--- a/ckanext/ga_report/command.py
+++ b/ckanext/ga_report/command.py
@@ -66,8 +66,7 @@
And where <time-period> is:
all - data for all time
latest - (default) just the 'latest' data
- YYYY-MM-DD - just data for all time periods going
- back to (and including) this date
+ YYYY-MM - just data for the specific month
"""
summary = __doc__.split('\n')[0]
usage = __doc__
@@ -96,6 +95,7 @@
elif time_period == 'latest':
downloader.latest()
else:
- since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
- downloader.since_date(since_date)
+ # The month to use
+ for_date = datetime.datetime.strptime(time_period, '%Y-%m')
+ downloader.specific_month(for_date)
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -1,6 +1,8 @@
+import re
import logging
import operator
-from ckan.lib.base import BaseController, c, render, request, response
+import collections
+from ckan.lib.base import BaseController, c, render, request, response, abort
import sqlalchemy
from sqlalchemy import func, cast, Integer
@@ -10,10 +12,10 @@
log = logging.getLogger('ckanext.ga-report')
-def _get_month_name(str):
+def _get_month_name(strdate):
import calendar
from time import strptime
- d = strptime('2012-10', '%Y-%m')
+ d = strptime(strdate, '%Y-%m')
return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)
@@ -21,7 +23,7 @@
months = []
vals = model.Session.query(cls.period_name).distinct().all()
for m in vals:
- months.append( (m[0], _get_month_name(m)))
+ months.append( (m[0], _get_month_name(m[0])))
return sorted(months, key=operator.itemgetter(0), reverse=True)
@@ -30,11 +32,11 @@
def csv(self, month):
import csv
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.period_name==month).\
- order_by('GA_Stat.stat_name, GA_Stat.key').all()
-
- response.headers['Content-disposition'] = 'attachment; filename=dgu_analytics_%s.csv' % (month)
+ q = model.Session.query(GA_Stat)
+ if month != 'all':
+ q = q.filter(GA_Stat.period_name==month)
+ entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all()
+
response.headers['Content-Type'] = "text/csv; charset=utf-8"
writer = csv.writer(response)
@@ -53,13 +55,46 @@
c.months = _month_details(GA_Stat)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
-
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.stat_name=='Totals').\
- filter(GA_Stat.period_name==c.month).all()
- c.global_totals = [(s.key, s.value) for s in entries ]
+ c.month_desc = 'all time'
+ c.month = request.params.get('month', '')
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name=='Totals')
+ if c.month:
+ q = q.filter(GA_Stat.period_name==c.month)
+ entries = q.order_by('ga_stat.key').all()
+
+ def clean_key(key, val):
+ if key in ['Average time on site', 'Pages per visit', 'Percent new visits']:
+ val = "%.2f" % round(float(val), 2)
+ if key == 'Average time on site':
+ mins, secs = divmod(float(val), 60)
+ hours, mins = divmod(mins, 60)
+ val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
+ key = '%s *' % key
+ if key in ['Bounces', 'Total pageviews']:
+ val = int(val)
+ return key, val
+
+ c.global_totals = []
+ if c.month:
+ for e in entries:
+ key, val = clean_key(e.key, e.value)
+ c.global_totals.append((key, val))
+ else:
+ d = collections.defaultdict(list)
+ for e in entries:
+ d[e.key].append(float(e.value))
+ for k, v in d.iteritems():
+ if k in ['Bounces', 'Total pageviews']:
+ v = sum(v)
+ else:
+ v = float(sum(v))/len(v)
+ key, val = clean_key(k,v)
+ c.global_totals.append((key, val))
+ c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0))
keys = {
'Browser versions': 'browsers',
@@ -69,12 +104,45 @@
'Country': 'country'
}
+ browser_version_re = re.compile("(.*)\((.*)\)")
for k, v in keys.iteritems():
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.stat_name==k).\
- filter(GA_Stat.period_name==c.month).\
- order_by('ga_stat.value::int desc').all()
- setattr(c, v, [(s.key, s.value) for s in entries ])
+
+ def clean_field(key):
+ if k != 'Browser versions':
+ return key
+ m = browser_version_re.match(key)
+ browser = m.groups()[0].strip()
+ ver = m.groups()[1]
+ parts = ver.split('.')
+ if len(parts) > 1:
+ if parts[1][0] == '0':
+ ver = parts[0]
+ else:
+ ver = "%s.%s" % (parts[0],parts[1])
+ if browser in ['Safari','Android Browser']: # Special case complex version nums
+ ver = parts[0]
+ if len(ver) > 2:
+ ver = "%s%sX" % (ver[0], ver[1])
+
+ return "%s (%s)" % (browser, ver,)
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name==k)
+ if c.month:
+ entries = []
+ q = q.filter(GA_Stat.period_name==c.month).\
+ order_by('ga_stat.value::int desc')
+
+ d = collections.defaultdict(int)
+ for e in q.all():
+ d[clean_field(e.key)] += int(e.value)
+ entries = []
+ for key, val in d.iteritems():
+ entries.append((key,val,))
+ entries = sorted(entries, key=operator.itemgetter(1), reverse=True)
+
+ setattr(c, v, [(k,v) for k,v in entries ])
+
return render('ga_report/site/index.html')
@@ -87,25 +155,37 @@
"""
def index(self):
+
# Get the month details by fetching distinct values and determining the
# month names from the values.
c.months = _month_details(GA_Url)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+ c.month = request.params.get('month', '')
+ c.month_desc = 'all time'
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
connection = model.Session.connection()
q = """
select department_id, sum(pageviews::int) views, sum(visitors::int) visits
from ga_url
- where department_id <> ''
- and not url like '/publisher/%%'
- and period_name=%s
- group by department_id order by views desc limit 20;
- """
+ where department_id <> ''"""
+ if c.month:
+ q = q + """
+ and period_name=%s
+ """
+ q = q + """
+ group by department_id order by views desc limit 20;
+ """
+
+ # Add this back (before and period_name =%s) if you want to ignore publisher
+ # homepage views
+ # and not url like '/publisher/%%'
+
c.top_publishers = []
res = connection.execute(q, c.month)
+
for row in res:
c.top_publishers.append((model.Group.get(row[0]), row[1], row[2]))
@@ -113,7 +193,11 @@
def read(self, id):
+ count = 20
+
c.publisher = model.Group.get(id)
+ if not c.publisher:
+ abort(404, 'A publisher with that name could not be found')
c.top_packages = [] # package, dataset_views in c.top_packages
# Get the month details by fetching distinct values and determining the
@@ -121,22 +205,50 @@
c.months = _month_details(GA_Url)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
-
- entry = model.Session.query(GA_Url).\
- filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\
- filter(GA_Url.period_name==c.month).first()
- c.publisher_page_views = entry.pageviews if entry else 0
-
- entries = model.Session.query(GA_Url).\
+ c.month = request.params.get('month', '')
+ if not c.month:
+ c.month_desc = 'all time'
+ else:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ c.publisher_page_views = 0
+ q = model.Session.query(GA_Url).\
+ filter(GA_Url.url=='/publisher/%s' % c.publisher.name)
+ if c.month:
+ entry = q.filter(GA_Url.period_name==c.month).first()
+ c.publisher_page_views = entry.pageviews if entry else 0
+ else:
+ for e in q.all():
+ c.publisher_page_views = c.publisher_page_views + int(e.pageviews)
+
+
+ q = model.Session.query(GA_Url).\
filter(GA_Url.department_id==c.publisher.name).\
- filter(GA_Url.period_name==c.month).\
- order_by('ga_url.pageviews::int desc')[:20]
- for entry in entries:
- if entry.url.startswith('/dataset/'):
+ filter(GA_Url.url.like('/dataset/%'))
+ if c.month:
+ q = q.filter(GA_Url.period_name==c.month)
+ q = q.order_by('ga_url.pageviews::int desc')
+
+ if c.month:
+ for entry in q[:count]:
p = model.Package.get(entry.url[len('/dataset/'):])
c.top_packages.append((p,entry.pageviews,entry.visitors))
+ else:
+ ds = {}
+ for entry in q.all():
+ if len(ds) >= count:
+ break
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in ds:
+ ds[p] = {'views':0, 'visits': 0}
+ ds[p]['views'] = ds[p]['views'] + int(entry.pageviews)
+ ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors)
+
+ results = []
+ for k, v in ds.iteritems():
+ results.append((k,v['views'],v['visits']))
+
+ c.top_packages = sorted(results, key=operator.itemgetter(1), reverse=True)
return render('ga_report/publisher/read.html')
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -21,8 +21,17 @@
self.profile_id = profile_id
- def all_(self):
- self.since_date(datetime.datetime(2010, 1, 1))
+ def specific_month(self, date):
+ import calendar
+
+ first_of_this_month = datetime.datetime(date.year, date.month, 1)
+ _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month))
+ last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month)
+ periods = ((date.strftime(FORMAT_MONTH),
+ last_day_of_month,
+ first_of_this_month, last_of_this_month),)
+ self.download_and_store(periods)
+
def latest(self):
if self.period == 'monthly':
@@ -37,13 +46,13 @@
self.download_and_store(periods)
- def since_date(self, since_date):
+ def for_date(self, for_date):
assert isinstance(since_date, datetime.datetime)
periods = [] # (period_name, period_complete_day, start_date, end_date)
if self.period == 'monthly':
first_of_the_months_until_now = []
- year = since_date.year
- month = since_date.month
+ year = for_date.year
+ month = for_date.month
now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1)
while True:
--- /dev/null
+++ b/ckanext/ga_report/helpers.py
@@ -1,1 +1,42 @@
+import logging
+import operator
+import ckan.lib.base as base
+import ckan.model as model
+_log = logging.getLogger(__name__)
+
+def most_popular_datasets(publisher, count=20):
+ from ckanext.ga_report.ga_model import GA_Url
+
+ if not publisher:
+ _log.error("No valid publisher passed to 'most_popular_datasets'")
+ return ""
+
+ datasets = {}
+ entries = model.Session.query(GA_Url).\
+ filter(GA_Url.department_id==publisher.name).\
+ filter(GA_Url.url.like('/dataset/%')).\
+ order_by('ga_url.pageviews::int desc').all()
+ for entry in entries:
+ if len(datasets) < count:
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in datasets:
+ datasets[p] = {'views':0, 'visits': 0}
+ datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
+ datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
+
+ results = []
+ for k, v in datasets.iteritems():
+ results.append((k,v['views'],v['visits']))
+
+ results = sorted(results, key=operator.itemgetter(1), reverse=True)
+
+ ctx = {
+ 'dataset_count': len(datasets),
+ 'datasets': results,
+
+ 'publisher': publisher
+ }
+
+ return base.render_snippet('ga_report/publisher/popular.html', **ctx)
+
--- a/ckanext/ga_report/plugin.py
+++ b/ckanext/ga_report/plugin.py
@@ -2,33 +2,32 @@
import ckan.lib.helpers as h
import ckan.plugins as p
from ckan.plugins import implements, toolkit
-#import gasnippet
-#import commands
-#import dbutil
log = logging.getLogger('ckanext.ga-report')
class GAReportPlugin(p.SingletonPlugin):
implements(p.IConfigurer, inherit=True)
implements(p.IRoutes, inherit=True)
+ implements(p.ITemplateHelpers, inherit=True)
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
toolkit.add_public_directory(config, 'public')
+ def get_helpers(self):
+ """
+ A dictionary of extra helpers that will be available to provide
+ ga report info to templates.
+ """
+ from ckanext.ga_report.helpers import most_popular_datasets
+ return {
+ 'ga_report_installed': lambda: True,
+ 'most_popular_datasets': most_popular_datasets,
+ }
+
def after_map(self, map):
map.connect(
- '/data/analytics',
- controller='ckanext.ga_report.controller:GaReport',
- action='index'
- )
- map.connect(
- '/data/analytics_{month}.csv',
- controller='ckanext.ga_report.controller:GaReport',
- action='csv'
- )
- map.connect(
- '/data/analytics/publisher/',
+ '/data/analytics/publisher',
controller='ckanext.ga_report.controller:GaPublisherReport',
action='index'
)
@@ -37,6 +36,16 @@
controller='ckanext.ga_report.controller:GaPublisherReport',
action='read'
)
+ map.connect(
+ '/data/analytics',
+ controller='ckanext.ga_report.controller:GaReport',
+ action='index'
+ )
+ map.connect(
+ '/data/analytics/data_{month}.csv',
+ controller='ckanext.ga_report.controller:GaReport',
+ action='csv'
+ )
return map
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/ga_util.html
@@ -1,1 +1,37 @@
+<html
+ xmlns="http://www.w3.org/1999/xhtml"
+ xmlns:i18n="http://genshi.edgewall.org/i18n"
+ xmlns:py="http://genshi.edgewall.org/"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ py:strip=""
+ >
+ <table py:def="publisher_list(groups)" class="groups">
+ <py:for each="group,title in groups">
+ <tr>
+ <td><a href="/publisher/${group.name}">${title}</a></td>
+ </tr>
+ </py:for>
+ </table>
+
+
+<div py:def="usage_nav(active_name,publisher)" id="minornavigation">
+ <div id="minornavigation-bg-left">
+ <div id="minornavigation-bg-right">
+ <ul class="nav nav-pills">
+ <li py:attrs="{'class': 'active' if active_name=='Site-wide' else None}"><a py:attrs="{'class': 'active' if active_name=='Site-wide' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/> Site-wide</a></li>
+ <li py:attrs="{'class': 'active' if active_name=='Publishers' else None}">
+ <a py:attrs="{'class': 'active' if active_name=='Publishers' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/> Publishers</a>
+ </li>
+ <li py:if="publisher" class="active">
+ <a class="active" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read', id=publisher.name)}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/>${publisher.title}</a>
+ </li>
+
+ </ul>
+ </div>
+ </div>
+</div>
+
+
+</html>
+
--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -2,22 +2,41 @@
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
+
+ <xi:include href="../ga_util.html" />
<py:def function="page_title">Publisher Analytics for ${g.site_title}</py:def>
<py:match path="primarysidebar">
<li class="widget-container boxed widget_text">
<h4>Publishers</h4>
+ <p>
+ Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits.
+ </p>
+ <p>
+ Note: this data does not include API calls.
+ </p>
+ </li>
+ <li class="widget-container boxed widget_text">
+ <h4>Download</h4>
+ <p><center>
+ <a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month or 'all')}">Download as CSV</a></center>
+ </p>
</li>
</py:match>
+
<div py:match="content">
- <h1>Publisher Analytics</h1>
- <h2>The top 20 publishers</h2>
+ <h1>Site Usage</h1>
+
+ ${usage_nav('Publishers', None)}
+
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" method="get">
<div class="controls">
<select name="month">
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
+
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
</py:for>
@@ -59,5 +78,3 @@
</html>
-
-
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/publisher/popular.html
@@ -1,1 +1,25 @@
+<html xmlns:py="http://genshi.edgewall.org/"
+ xmlns:i18n="http://genshi.edgewall.org/i18n"
+ xmlns:xi="http://www.w3.org/2001/XInclude"
+ py:strip="">
+ <py:if test="dataset_count == 0">
+ <p>We do not currently have analytics data for ${publisher.title}</p>
+ </py:if>
+ <py:if test="dataset_count > 0">
+ <div class="popular_datasets">
+ <ul>
+ <py:for each="dataset,views,visits in datasets">
+ <li>
+ ${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))}
+ </li>
+ </py:for>
+ </ul>
+ <p class="">${h.link_to("More analytics for " + publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=publisher.name))}</p>
+ </div>
+ </py:if>
+</html>
+
+
+
+
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -3,33 +3,39 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
+ <xi:include href="../ga_util.html" />
+
<py:def function="page_title">Analytics for ${g.site_title}</py:def>
- <py:match path="primarysidebar">
+ <py:match path="primarysidebar">
<li class="widget-container boxed widget_text">
- <h4>${c.publisher.title}</h4>
+ <h4>Publishers</h4>
<p>
- The table shows the top 20 most viewed datasets belonging to ${c.publisher.title}.
+ Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits.
</p>
<p>
- As well as showing the number of views within ${c.month_desc}, it will also show the
- number of visitors that viewed each dataset.
+ Note: this data does not include API calls.
</p>
- <p>
- <p>The dataset list page for <a href="${h.url_for(controller='ckanext.dgu.controllers.publisher:PublisherController', action='read', id=c.publisher.name)}">${c.publisher.title}</a> was viewed ${c.publisher_page_views} times during ${c.month_desc}</p>
+ </li>
+
+ <li class="widget-container boxed widget_text">
+ <h4>Download</h4>
+ <p><center>
+ <a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month or 'all')}">Download as CSV</a></center>
</p>
</li>
</py:match>
+
<div py:match="content">
- <h1>Analytics for ${c.publisher.title}</h1>
+ <h1>Site Usage</h1>
- <h2>Top 20 most viewed datasets</h2>
- <p><em>Note: this data does not include API calls</em></p>
+ ${usage_nav(c.publisher.title, c.publisher)}
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=c.publisher.name)}" method="get">
<div class="controls">
<select name="month">
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
</py:for>
--- a/ckanext/ga_report/templates/ga_report/site/index.html
+++ b/ckanext/ga_report/templates/ga_report/site/index.html
@@ -3,22 +3,36 @@
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
- <py:def function="page_title">Site analytics</py:def>
+ <xi:include href="../ga_util.html" />
+
+ <py:def function="page_title">Site usage</py:def>
<py:match path="primarysidebar">
<li class="widget-container boxed widget_text">
- <h4>Statistics</h4>
- <p>It is possible to <a href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month)}">export the analytics data</a> as a CSV file, which contains all of the information for ${c.month_desc}</p>
+ <h4>Site-wide</h4>
+ <p>
+ Note: this data does not include API calls and some values have been rounded up to 2 decimal places. Where there are a large number of browser versions they have been grouped together.
+ </p>
+ </li>
+ <li class="widget-container boxed widget_text">
+ <h4>Download</h4>
+ <p><center>
+ <a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month or 'all')}">Download as CSV</a></center>
+ </p>
+ </li>
- </li>
</py:match>
<div py:match="content">
- <h1>Site statistics</h1>
+ <h1>Site Usage</h1>
+
+ ${usage_nav('Site-wide', None)}
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<div class="controls">
<select name="month">
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
+
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
</py:for>
--- a/ckanext/ga_report/tests/test_api.py
+++ b/ckanext/ga_report/tests/test_api.py
@@ -32,7 +32,7 @@
svc = init_service("token.dat", "credentials.json")
downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
try:
- downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30))
+ downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30))
except Exception as e:
assert False, e