Rename of all time to all months
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -4,12 +4,12 @@
import logging
import operator
import collections
-from ckan.lib.base import BaseController, c, render, request, response, abort
+from ckan.lib.base import (BaseController, c, g, render, request, response, abort)
import sqlalchemy
from sqlalchemy import func, cast, Integer
import ckan.model as model
-from ga_model import GA_Url, GA_Stat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat
log = logging.getLogger('ckanext.ga-report')
@@ -58,7 +58,7 @@
c.months = _month_details(GA_Stat)
# Work out which month to show, based on query params of the first item
- c.month_desc = 'all time'
+ c.month_desc = 'all months'
c.month = request.params.get('month', '')
if c.month:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
@@ -70,15 +70,17 @@
entries = q.order_by('ga_stat.key').all()
def clean_key(key, val):
- if key in ['Average time on site', 'Pages per visit', 'Percent new visits']:
+ if key in ['Average time on site', 'Pages per visit', 'New visits']:
val = "%.2f" % round(float(val), 2)
if key == 'Average time on site':
mins, secs = divmod(float(val), 60)
hours, mins = divmod(mins, 60)
val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
- key = '%s *' % key
- if key in ['Bounces', 'Total pageviews']:
+ if key == 'New visits':
+ val = "%s%%" % val
+ if key in ['Bounces', 'Total page views', 'Total visits']:
val = int(val)
+
return key, val
c.global_totals = []
@@ -91,7 +93,7 @@
for e in entries:
d[e.key].append(float(e.value))
for k, v in d.iteritems():
- if k in ['Bounces', 'Total pageviews']:
+ if k in ['Bounces', 'Total page views', 'Total visits']:
v = sum(v)
else:
v = float(sum(v))/len(v)
@@ -100,12 +102,38 @@
c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0))
keys = {
- 'Browser versions': 'browsers',
- 'Operating Systems versions': 'os',
+ 'Browser versions': 'browser_versions',
+ 'Browsers': 'browsers',
+ 'Operating Systems versions': 'os_versions',
+ 'Operating Systems': 'os',
'Social sources': 'social_networks',
'Languages': 'languages',
'Country': 'country'
}
+
+ def shorten_name(name, length=60):
+ return (name[:length] + '..') if len(name) > 60 else name
+
+ def fill_out_url(url):
+ import urlparse
+ return urlparse.urljoin(g.site_url, url)
+
+ c.social_referrer_totals, c.social_referrers = [], []
+ q = model.Session.query(GA_ReferralStat)
+ q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
+ q = q.order_by('ga_referrer.count::int desc')
+ for entry in q.all():
+ c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url),
+ entry.source,entry.count))
+
+ q = model.Session.query(GA_ReferralStat.url,
+ func.sum(GA_ReferralStat.count).label('count'))
+ q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
+ q = q.order_by('count desc').group_by(GA_ReferralStat.url)
+ for entry in q.all():
+ c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'',
+ entry[1]))
+
browser_version_re = re.compile("(.*)\((.*)\)")
for k, v in keys.iteritems():
@@ -144,9 +172,17 @@
entries.append((key,val,))
entries = sorted(entries, key=operator.itemgetter(1), reverse=True)
- setattr(c, v, [(k,v) for k,v in entries ])
-
-
+ def percent(num, total):
+ p = 100 * float(num)/float(total)
+ return "%.2f%%" % round(p, 2)
+
+ # Get the total for each set of values and then set the value as
+ # a percentage of the total
+ if k == 'Social sources':
+ total = sum([x for n,x in c.global_totals if n == 'Total visits'])
+ else:
+ total = sum([num for _,num in entries])
+ setattr(c, v, [(k,percent(v,total)) for k,v in entries ])
return render('ga_report/site/index.html')
@@ -204,7 +240,7 @@
# Work out which month to show, based on query params of the first item
c.month = request.params.get('month', '')
- c.month_desc = 'all time'
+ c.month_desc = 'all months'
if c.month:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
@@ -291,7 +327,7 @@
# Work out which month to show, based on query params of the first item
c.month = request.params.get('month', '')
if not c.month:
- c.month_desc = 'all time'
+ c.month_desc = 'all months'
else:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,7 +1,7 @@
import os
import logging
import datetime
-
+import collections
from pylons import config
import ga_model
@@ -94,7 +94,6 @@
self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d'))
-
data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
log.info('Storing Dataset Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day))
@@ -107,6 +106,33 @@
ga_model.update_publisher_stats(period_name) # about 30 seconds.
self.sitewide_stats( period_name )
+
+ self.update_social_info(period_name, start_date, end_date)
+
+ def update_social_info(self, period_name, start_date, end_date):
+ start_date = start_date.strftime('%Y-%m-%d')
+ end_date = end_date.strftime('%Y-%m-%d')
+ query = 'ga:hasSocialSourceReferral=~Yes$'
+ metrics = 'ga:entrances'
+ sort = '-ga:entrances'
+
+ # Supported query params at
+ # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ filters=query,
+ start_date=start_date,
+ metrics=metrics,
+ sort=sort,
+ dimensions="ga:landingPagePath,ga:socialNetwork",
+ max_results=10000,
+ end_date=end_date).execute()
+ data = collections.defaultdict(list)
+ rows = results.get('rows',[])
+ for row in rows:
+ from ga_model import _normalize_url
+ data[_normalize_url(row[0])].append( (row[1], int(row[2]),) )
+ ga_model.update_social(period_name, data)
def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
@@ -176,12 +202,12 @@
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
- ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]})
-
- results = self.service.data().ga().get(
- ids='ga:' + self.profile_id,
- start_date=start_date,
- metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits',
+ ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]})
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors',
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
@@ -189,7 +215,8 @@
'Pages per visit': result_data[0][0],
'Bounces': result_data[0][1],
'Average time on site': result_data[0][2],
- 'Percent new visits': result_data[0][3],
+ 'New visits': result_data[0][3],
+ 'Total visits': result_data[0][4],
}
ga_model.update_sitewide_stats(period_name, "Totals", data)
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -13,6 +13,8 @@
def make_uuid():
return unicode(uuid.uuid4())
+metadata = MetaData()
+
class GA_Url(object):
@@ -21,20 +23,6 @@
for k,v in kwargs.items():
setattr(self, k, v)
-class GA_Stat(object):
-
- def __init__(self, **kwargs):
- for k,v in kwargs.items():
- setattr(self, k, v)
-
-class GA_Publisher(object):
-
- def __init__(self, **kwargs):
- for k,v in kwargs.items():
- setattr(self, k, v)
-
-
-metadata = MetaData()
url_table = Table('ga_url', metadata,
Column('id', types.UnicodeText, primary_key=True,
default=make_uuid),
@@ -47,6 +35,13 @@
)
mapper(GA_Url, url_table)
+
+class GA_Stat(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
stat_table = Table('ga_stat', metadata,
Column('id', types.UnicodeText, primary_key=True,
default=make_uuid),
@@ -56,6 +51,12 @@
Column('value', types.UnicodeText), )
mapper(GA_Stat, stat_table)
+
+class GA_Publisher(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
pub_table = Table('ga_publisher', metadata,
Column('id', types.UnicodeText, primary_key=True,
@@ -71,6 +72,24 @@
mapper(GA_Publisher, pub_table)
+class GA_ReferralStat(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+referrer_table = Table('ga_referrer', metadata,
+ Column('id', types.UnicodeText, primary_key=True,
+ default=make_uuid),
+ Column('period_name', types.UnicodeText),
+ Column('source', types.UnicodeText),
+ Column('url', types.UnicodeText),
+ Column('count', types.Integer),
+ )
+mapper(GA_ReferralStat, referrer_table)
+
+
+
def init_tables():
metadata.create_all(model.meta.engine)
@@ -93,8 +112,9 @@
>>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices')
'/dataset/weekly_fuel_prices'
'''
- url = re.sub('https?://(www\.)?data.gov.uk', '', url)
- return url
+ # Deliberately leaving a /
+ url = url.replace('http:/','')
+ return '/' + '/'.join(url.split('/')[2:])
def _get_department_id_of_url(url):
@@ -167,6 +187,33 @@
model.Session.commit()
+def update_social(period_name, data):
+ # Clean up first.
+ model.Session.query(GA_ReferralStat).\
+ filter(GA_ReferralStat.period_name==period_name).delete()
+
+ for url,data in data.iteritems():
+ for entry in data:
+ source = entry[0]
+ count = entry[1]
+
+ item = model.Session.query(GA_ReferralStat).\
+ filter(GA_ReferralStat.period_name==period_name).\
+ filter(GA_ReferralStat.source==source).\
+ filter(GA_ReferralStat.url==url).first()
+ if item:
+ item.count = item.count + count
+ model.Session.add(item)
+ else:
+ # create the row
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'source': source,
+ 'url': url,
+ 'count': count,
+ }
+ model.Session.add(GA_ReferralStat(**values))
+ model.Session.commit()
def update_publisher_stats(period_name):
"""
--- a/ckanext/ga_report/templates/ga_report/ga_util.html
+++ b/ckanext/ga_report/templates/ga_report/ga_util.html
@@ -12,6 +12,36 @@
<td><a href="/publisher/${group.name}">${title}</a></td>
</tr>
</py:for>
+ </table>
+
+
+<table py:def="social_table(items, with_source=False)" class="table table-condensed table-bordered table-striped">
+ <tr>
+ <th>Name</th>
+ <th py:if="with_source">Source</th>
+ <th>Visits</th>
+ </tr>
+ <py:for each="name, url, source, count in items">
+ <tr>
+ <td><a href="${url}">${name}</a></td>
+ <td py:if="with_source">${source}</td>
+ <td>${count}</td>
+ </tr>
+ </py:for>
+ </table>
+
+
+<table py:def="stat_table(items, title='Views')" class="table table-condensed table-bordered table-striped">
+ <tr>
+ <th>Name</th>
+ <th>% ${title}</th>
+ </tr>
+ <py:for each="name, value in items">
+ <tr>
+ <td>${name}</td>
+ <td>${value}</td>
+ </tr>
+ </py:for>
</table>
--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -35,7 +35,7 @@
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" method="get">
<div class="controls">
<select name="month">
- <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option>
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -35,7 +35,7 @@
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=c.publisher.name)}" method="get">
<div class="controls">
<select name="month">
- <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option>
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
</py:for>
--- a/ckanext/ga_report/templates/ga_report/site/index.html
+++ b/ckanext/ga_report/templates/ga_report/site/index.html
@@ -25,13 +25,12 @@
<div py:match="content">
<h1>Site Usage</h1>
-
${usage_nav('Site-wide', None)}
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<div class="controls">
<select name="month">
- <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All time</option>
+ <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option>
<py:for each="val,desc in c.months">
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option>
@@ -44,9 +43,32 @@
<div class="tabbable">
<ul class="nav nav-tabs">
<li class="active"><a href="#totals" data-toggle="tab">Totals</a></li>
- <li><a href="#browsers" data-toggle="tab">Browsers</a></li>
- <li><a href="#os" data-toggle="tab">Operating Systems</a></li>
- <li><a href="#social_networks" data-toggle="tab">Social Networks</a></li>
+ <li class="dropdown">
+ <a href="#browsers" class="dropdown-toggle" data-toggle="dropdown">Browsers
+ <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="#browsers_names" data-toggle="tab">Browsers</a></li>
+ <li><a href="#browsers_versions" data-toggle="tab">Versions</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#browsers" class="dropdown-toggle" data-toggle="dropdown">Operating<br/>Systems
+ <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="#os" data-toggle="tab">Operating Systems</a></li>
+ <li><a href="#os_versions" data-toggle="tab">Versions</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#browsers" class="dropdown-toggle" data-toggle="dropdown">Social Networks
+ <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><a href="#social_networks" data-toggle="tab">All networks</a></li>
+ <li><a href="#social_referrals_totals" data-toggle="tab">Referral links</a></li>
+ </ul>
+ </li>
+
+ <li><a href="#social_networks" data-toggle="tab"></a></li>
<li><a href="#languages" data-toggle="tab">Languages</a></li>
<li><a href="#country" data-toggle="tab">Country</a></li>
</ul>
@@ -65,75 +87,32 @@
</py:for>
</table>
</div>
- <div class="tab-pane" id="browsers">
- <table class="table table-condensed table-bordered table-striped">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- <py:for each="name, value in c.browsers">
- <tr>
- <td>${name}</td>
- <td>${value}</td>
- </tr>
- </py:for>
- </table>
- </div>
- <div class="tab-pane" id="os">
- <table class="table table-condensed table-bordered table-striped">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- <py:for each="name, value in c.os">
- <tr>
- <td>${name}</td>
- <td>${value}</td>
- </tr>
- </py:for>
- </table>
+ <div class="tab-pane" id="browsers_versions">
+ ${stat_table(c.browser_versions)}
+ </div>
+ <div class="tab-pane" id="browsers_names">
+ ${stat_table(c.browsers)}
+ </div>
+ <div class="tab-pane" id="os">
+ ${stat_table(c.os)}
+ </div>
+ <div class="tab-pane" id="os_versions">
+ ${stat_table(c.os_versions)}
+ </div>
+ <div class="tab-pane" id="social_referrals_totals">
+ <p>Number of visits to urls referred from social networks</p>
+ ${social_table(c.social_referrer_totals)}
</div>
<div class="tab-pane" id="social_networks">
- <table class="table table-condensed table-bordered table-striped">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- <py:for each="name, value in c.social_networks">
- <tr>
- <td>${name}</td>
- <td>${value}</td>
- </tr>
- </py:for>
- </table>
+ <p>Percentage of visits referred from these social networks</p>
+
+ ${stat_table(c.social_networks, 'Visits')}
</div>
<div class="tab-pane" id="languages">
- <table class="table table-condensed table-bordered table-striped">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- <py:for each="name, value in c.languages">
- <tr>
- <td>${name}</td>
- <td>${value}</td>
- </tr>
- </py:for>
- </table>
+ ${stat_table(c.languages)}
</div>
<div class="tab-pane" id="country">
- <table class="table table-condensed table-bordered table-striped">
- <tr>
- <th>Name</th>
- <th>Value</th>
- </tr>
- <py:for each="name, value in c.country">
- <tr>
- <td>${name}</td>
- <td>${value}</td>
- </tr>
- </py:for>
- </table>
+ ${stat_table(c.country)}
</div>
@@ -148,10 +127,12 @@
<py:def function="optional_footer">
<script type='text/javascript'>
+ $('.dropdown-toggle').dropdown();
$('.nav-tabs li a').click(function (e) {
e.preventDefault();
$(this).tab('show');
})
+ alert(window.location.hash);
</script>
</py:def>
</html>