From: Ross Jones
Date: Thu, 01 Nov 2012 19:10:03 +0000
Subject: Adding back a link to the publisher
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=64814c1997f252dd673d6494f62b15c2a449d929
---
Adding back a link to the publisher
---
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -1,11 +1,15 @@
+import re
+import csv
+import sys
import logging
import operator
-from ckan.lib.base import BaseController, c, render, request, response, abort
+import collections
+from ckan.lib.base import (BaseController, c, g, render, request, response, abort)
import sqlalchemy
from sqlalchemy import func, cast, Integer
import ckan.model as model
-from ga_model import GA_Url, GA_Stat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat
log = logging.getLogger('ckanext.ga-report')
@@ -30,11 +34,13 @@
def csv(self, month):
import csv
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.period_name==month).\
- order_by('GA_Stat.stat_name, GA_Stat.key').all()
+ q = model.Session.query(GA_Stat)
+ if month != 'all':
+ q = q.filter(GA_Stat.period_name==month)
+ entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all()
response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,))
writer = csv.writer(response)
writer.writerow(["Period", "Statistic", "Key", "Value"])
@@ -52,42 +58,131 @@
c.months = _month_details(GA_Stat)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
-
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.stat_name=='Totals').\
- filter(GA_Stat.period_name==c.month).\
- order_by('ga_stat.key').all()
-
- c.global_totals = []
- for e in entries:
- val = e.value
- if e.key in ['Average time on site', 'Pages per visit', 'Percent new visits']:
- val = "%.2f" % round(float(e.value), 2)
- if e.key == 'Average time on site':
+ c.month_desc = 'all months'
+ c.month = request.params.get('month', '')
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name=='Totals')
+ if c.month:
+ q = q.filter(GA_Stat.period_name==c.month)
+ entries = q.order_by('ga_stat.key').all()
+
+ def clean_key(key, val):
+ if key in ['Average time on site', 'Pages per visit', 'New visits']:
+ val = "%.2f" % round(float(val), 2)
+ if key == 'Average time on site':
mins, secs = divmod(float(val), 60)
hours, mins = divmod(mins, 60)
val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
- e.key = '%s *' % e.key
- c.global_totals.append((e.key, val))
-
+ if key == 'New visits':
+ val = "%s%%" % val
+ if key in ['Bounces', 'Total page views', 'Total visits']:
+ val = int(val)
+
+ return key, val
+
+ c.global_totals = []
+ if c.month:
+ for e in entries:
+ key, val = clean_key(e.key, e.value)
+ c.global_totals.append((key, val))
+ else:
+ d = collections.defaultdict(list)
+ for e in entries:
+ d[e.key].append(float(e.value))
+ for k, v in d.iteritems():
+ if k in ['Bounces', 'Total page views', 'Total visits']:
+ v = sum(v)
+ else:
+ v = float(sum(v))/len(v)
+ key, val = clean_key(k,v)
+ c.global_totals.append((key, val))
+ c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0))
keys = {
- 'Browser versions': 'browsers',
- 'Operating Systems versions': 'os',
+ 'Browser versions': 'browser_versions',
+ 'Browsers': 'browsers',
+ 'Operating Systems versions': 'os_versions',
+ 'Operating Systems': 'os',
'Social sources': 'social_networks',
'Languages': 'languages',
'Country': 'country'
}
+ def shorten_name(name, length=60):
+ return (name[:length] + '..') if len(name) > 60 else name
+
+ def fill_out_url(url):
+ import urlparse
+ return urlparse.urljoin(g.site_url, url)
+
+ c.social_referrer_totals, c.social_referrers = [], []
+ q = model.Session.query(GA_ReferralStat)
+ q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
+ q = q.order_by('ga_referrer.count::int desc')
+ for entry in q.all():
+ c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url),
+ entry.source,entry.count))
+
+ q = model.Session.query(GA_ReferralStat.url,
+ func.sum(GA_ReferralStat.count).label('count'))
+ q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
+ q = q.order_by('count desc').group_by(GA_ReferralStat.url)
+ for entry in q.all():
+ c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'',
+ entry[1]))
+
+
+ browser_version_re = re.compile("(.*)\((.*)\)")
for k, v in keys.iteritems():
- entries = model.Session.query(GA_Stat).\
- filter(GA_Stat.stat_name==k).\
- filter(GA_Stat.period_name==c.month).\
- order_by('ga_stat.value::int desc').all()
- setattr(c, v, [(s.key, s.value) for s in entries ])
-
+
+ def clean_field(key):
+ if k != 'Browser versions':
+ return key
+ m = browser_version_re.match(key)
+ browser = m.groups()[0].strip()
+ ver = m.groups()[1]
+ parts = ver.split('.')
+ if len(parts) > 1:
+ if parts[1][0] == '0':
+ ver = parts[0]
+ else:
+ ver = "%s.%s" % (parts[0],parts[1])
+ if browser in ['Safari','Android Browser']: # Special case complex version nums
+ ver = parts[0]
+ if len(ver) > 2:
+ ver = "%s%sX" % (ver[0], ver[1])
+
+ return "%s (%s)" % (browser, ver,)
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name==k)
+ if c.month:
+ entries = []
+ q = q.filter(GA_Stat.period_name==c.month).\
+ order_by('ga_stat.value::int desc')
+
+ d = collections.defaultdict(int)
+ for e in q.all():
+ d[clean_field(e.key)] += int(e.value)
+ entries = []
+ for key, val in d.iteritems():
+ entries.append((key,val,))
+ entries = sorted(entries, key=operator.itemgetter(1), reverse=True)
+
+ def percent(num, total):
+ p = 100 * float(num)/float(total)
+ return "%.2f%%" % round(p, 2)
+
+ # Get the total for each set of values and then set the value as
+ # a percentage of the total
+ if k == 'Social sources':
+ total = sum([x for n,x in c.global_totals if n == 'Total visits'])
+ else:
+ total = sum([num for _,num in entries])
+ setattr(c, v, [(k,percent(v,total)) for k,v in entries ])
return render('ga_report/site/index.html')
@@ -97,6 +192,45 @@
Displays the pageview and visit count for specific publishers based on
the datasets associated with the publisher.
"""
+ def csv(self, month):
+
+ c.month = month if not month =='all' else ''
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,))
+
+ writer = csv.writer(response)
+ writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
+
+ for publisher,view,visit in self._get_publishers(None):
+ writer.writerow([publisher.title.encode('utf-8'),
+ view,
+ visit,
+ month])
+
+
+
+ def publisher_csv(self, id, month):
+
+ c.month = month if not month =='all' else ''
+ c.publisher = model.Group.get(id)
+ if not c.publisher:
+ abort(404, 'A publisher with that name could not be found')
+
+ packages = self._get_packages(c.publisher)
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = \
+ str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,))
+
+ writer = csv.writer(response)
+ writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
+
+ for package,view,visit in packages:
+ writer.writerow([package.title.encode('utf-8'),
+ view,
+ visit,
+ month])
+
+
def index(self):
@@ -105,30 +239,81 @@
c.months = _month_details(GA_Url)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
-
+ c.month = request.params.get('month', '')
+ c.month_desc = 'all months'
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ c.top_publishers = self._get_publishers()
+
+ return render('ga_report/publisher/index.html')
+
+ def _get_publishers(self, limit=20):
connection = model.Session.connection()
q = """
select department_id, sum(pageviews::int) views, sum(visitors::int) visits
from ga_url
- where department_id <> ''
- and period_name=%s
- group by department_id order by views desc limit 20;
- """
+ where department_id <> ''"""
+ if c.month:
+ q = q + """
+ and period_name=%s
+ """
+ q = q + """
+ group by department_id order by views desc
+ """
+ if limit:
+ q = q + " limit %s;" % (limit)
+
# Add this back (before and period_name =%s) if you want to ignore publisher
# homepage views
# and not url like '/publisher/%%'
- c.top_publishers = []
+ top_publishers = []
res = connection.execute(q, c.month)
+
for row in res:
- c.top_publishers.append((model.Group.get(row[0]), row[1], row[2]))
-
- return render('ga_report/publisher/index.html')
+ g = model.Group.get(row[0])
+ if g:
+ top_publishers.append((g, row[1], row[2]))
+ return top_publishers
+
+ def _get_packages(self, publisher, count=-1):
+ if count == -1:
+ count = sys.maxint
+
+ top_packages = []
+ q = model.Session.query(GA_Url).\
+ filter(GA_Url.department_id==publisher.name).\
+ filter(GA_Url.url.like('/dataset/%'))
+ if c.month:
+ q = q.filter(GA_Url.period_name==c.month)
+ q = q.order_by('ga_url.pageviews::int desc')
+
+ if c.month:
+ for entry in q[:count]:
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ top_packages.append((p,entry.pageviews,entry.visitors))
+ else:
+ ds = {}
+ for entry in q.all():
+ if len(ds) >= count:
+ break
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in ds:
+ ds[p] = {'views':0, 'visits': 0}
+ ds[p]['views'] = ds[p]['views'] + int(entry.pageviews)
+ ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors)
+
+ results = []
+ for k, v in ds.iteritems():
+ results.append((k,v['views'],v['visits']))
+
+ top_packages = sorted(results, key=operator.itemgetter(1), reverse=True)
+ return top_packages
def read(self, id):
+ count = 20
c.publisher = model.Group.get(id)
if not c.publisher:
@@ -140,22 +325,23 @@
c.months = _month_details(GA_Url)
# Work out which month to show, based on query params of the first item
- c.month = request.params.get('month', c.months[0][0] if c.months else '')
- c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
-
- entry = model.Session.query(GA_Url).\
- filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\
- filter(GA_Url.period_name==c.month).first()
- c.publisher_page_views = entry.pageviews if entry else 0
-
- entries = model.Session.query(GA_Url).\
- filter(GA_Url.department_id==c.publisher.name).\
- filter(GA_Url.period_name==c.month).\
- order_by('ga_url.pageviews::int desc')[:20]
- for entry in entries:
- if entry.url.startswith('/dataset/'):
- p = model.Package.get(entry.url[len('/dataset/'):])
- c.top_packages.append((p,entry.pageviews,entry.visitors))
+ c.month = request.params.get('month', '')
+ if not c.month:
+ c.month_desc = 'all months'
+ else:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ c.publisher_page_views = 0
+ q = model.Session.query(GA_Url).\
+ filter(GA_Url.url=='/publisher/%s' % c.publisher.name)
+ if c.month:
+ entry = q.filter(GA_Url.period_name==c.month).first()
+ c.publisher_page_views = entry.pageviews if entry else 0
+ else:
+ for e in q.all():
+ c.publisher_page_views = c.publisher_page_views + int(e.pageviews)
+
+ c.top_packages = self._get_packages(c.publisher, 20)
return render('ga_report/publisher/read.html')
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,7 +1,7 @@
import os
import logging
import datetime
-
+import collections
from pylons import config
import ga_model
@@ -94,7 +94,6 @@
self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d'))
-
data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
log.info('Storing Dataset Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day))
@@ -107,6 +106,33 @@
ga_model.update_publisher_stats(period_name) # about 30 seconds.
self.sitewide_stats( period_name )
+
+ self.update_social_info(period_name, start_date, end_date)
+
+ def update_social_info(self, period_name, start_date, end_date):
+ start_date = start_date.strftime('%Y-%m-%d')
+ end_date = end_date.strftime('%Y-%m-%d')
+ query = 'ga:hasSocialSourceReferral=~Yes$'
+ metrics = 'ga:entrances'
+ sort = '-ga:entrances'
+
+ # Supported query params at
+ # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ filters=query,
+ start_date=start_date,
+ metrics=metrics,
+ sort=sort,
+ dimensions="ga:landingPagePath,ga:socialNetwork",
+ max_results=10000,
+ end_date=end_date).execute()
+ data = collections.defaultdict(list)
+ rows = results.get('rows',[])
+ for row in rows:
+ from ga_model import _normalize_url
+ data[_normalize_url(row[0])].append( (row[1], int(row[2]),) )
+ ga_model.update_social(period_name, data)
def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
@@ -176,12 +202,12 @@
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
- ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]})
-
- results = self.service.data().ga().get(
- ids='ga:' + self.profile_id,
- start_date=start_date,
- metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits',
+ ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]})
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors',
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
@@ -189,7 +215,8 @@
'Pages per visit': result_data[0][0],
'Bounces': result_data[0][1],
'Average time on site': result_data[0][2],
- 'Percent new visits': result_data[0][3],
+ 'New visits': result_data[0][3],
+ 'Total visits': result_data[0][4],
}
ga_model.update_sitewide_stats(period_name, "Totals", data)
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -13,6 +13,8 @@
def make_uuid():
return unicode(uuid.uuid4())
+metadata = MetaData()
+
class GA_Url(object):
@@ -21,20 +23,6 @@
for k,v in kwargs.items():
setattr(self, k, v)
-class GA_Stat(object):
-
- def __init__(self, **kwargs):
- for k,v in kwargs.items():
- setattr(self, k, v)
-
-class GA_Publisher(object):
-
- def __init__(self, **kwargs):
- for k,v in kwargs.items():
- setattr(self, k, v)
-
-
-metadata = MetaData()
url_table = Table('ga_url', metadata,
Column('id', types.UnicodeText, primary_key=True,
default=make_uuid),
@@ -47,6 +35,13 @@
)
mapper(GA_Url, url_table)
+
+class GA_Stat(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
stat_table = Table('ga_stat', metadata,
Column('id', types.UnicodeText, primary_key=True,
default=make_uuid),
@@ -56,6 +51,12 @@
Column('value', types.UnicodeText), )
mapper(GA_Stat, stat_table)
+
+class GA_Publisher(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
pub_table = Table('ga_publisher', metadata,
Column('id', types.UnicodeText, primary_key=True,
@@ -71,6 +72,24 @@
mapper(GA_Publisher, pub_table)
+class GA_ReferralStat(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+referrer_table = Table('ga_referrer', metadata,
+ Column('id', types.UnicodeText, primary_key=True,
+ default=make_uuid),
+ Column('period_name', types.UnicodeText),
+ Column('source', types.UnicodeText),
+ Column('url', types.UnicodeText),
+ Column('count', types.Integer),
+ )
+mapper(GA_ReferralStat, referrer_table)
+
+
+
def init_tables():
metadata.create_all(model.meta.engine)
@@ -93,8 +112,9 @@
>>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices')
'/dataset/weekly_fuel_prices'
'''
- url = re.sub('https?://(www\.)?data.gov.uk', '', url)
- return url
+ # Deliberately leaving a /
+ url = url.replace('http:/','')
+ return '/' + '/'.join(url.split('/')[2:])
def _get_department_id_of_url(url):
@@ -167,6 +187,33 @@
model.Session.commit()
+def update_social(period_name, data):
+ # Clean up first.
+ model.Session.query(GA_ReferralStat).\
+ filter(GA_ReferralStat.period_name==period_name).delete()
+
+ for url,data in data.iteritems():
+ for entry in data:
+ source = entry[0]
+ count = entry[1]
+
+ item = model.Session.query(GA_ReferralStat).\
+ filter(GA_ReferralStat.period_name==period_name).\
+ filter(GA_ReferralStat.source==source).\
+ filter(GA_ReferralStat.url==url).first()
+ if item:
+ item.count = item.count + count
+ model.Session.add(item)
+ else:
+ # create the row
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'source': source,
+ 'url': url,
+ 'count': count,
+ }
+ model.Session.add(GA_ReferralStat(**values))
+ model.Session.commit()
def update_publisher_stats(period_name):
"""
--- a/ckanext/ga_report/helpers.py
+++ b/ckanext/ga_report/helpers.py
@@ -16,13 +16,14 @@
entries = model.Session.query(GA_Url).\
filter(GA_Url.department_id==publisher.name).\
filter(GA_Url.url.like('/dataset/%')).\
- order_by('ga_url.pageviews::int desc')[:count]
+ order_by('ga_url.pageviews::int desc').all()
for entry in entries:
- p = model.Package.get(entry.url[len('/dataset/'):])
- if not p in datasets:
- datasets[p] = {'views':0, 'visits': 0}
- datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
- datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
+ if len(datasets) < count:
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in datasets:
+ datasets[p] = {'views':0, 'visits': 0}
+ datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
+ datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
results = []
for k, v in datasets.iteritems():
--- a/ckanext/ga_report/plugin.py
+++ b/ckanext/ga_report/plugin.py
@@ -27,22 +27,32 @@
def after_map(self, map):
map.connect(
- '/data/analytics/publisher',
+ '/data/site-usage/publisher',
controller='ckanext.ga_report.controller:GaPublisherReport',
action='index'
)
map.connect(
- '/data/analytics/publisher/{id}',
+ '/data/site-usage/publisher_{month}.csv',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
+ action='csv'
+ )
+ map.connect(
+ '/data/site-usage/publisher/{id}_{month}.csv',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
+ action='publisher_csv'
+ )
+ map.connect(
+ '/data/site-usage/publisher/{id}',
controller='ckanext.ga_report.controller:GaPublisherReport',
action='read'
)
map.connect(
- '/data/analytics',
+ '/data/site-usage',
controller='ckanext.ga_report.controller:GaReport',
action='index'
)
map.connect(
- '/data/analytics/data_{month}.csv',
+ '/data/site-usage/data_{month}.csv',
controller='ckanext.ga_report.controller:GaReport',
action='csv'
)
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/ga_util.html
@@ -1,1 +1,67 @@
+
+
+
+
+
+
+ Name |
+ Source |
+ Visits |
+
+
+
+ ${name} |
+ ${source} |
+ ${count} |
+
+
+
+
+
+
+
+ Name |
+ % ${title} |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+
+
+
--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -2,23 +2,41 @@
xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">
+
+
Publisher Analytics for ${g.site_title}
Publishers
- The table shows the top 20 publishers as recorded by page views of datasets owned by that publisher, and the number of visits to each publisher's home page.
+
+ Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits.
+
+
+ Note: this data does not include API calls.
+
+
+
+ Download
+
+ Download as CSV
+
+
-
Publisher Analytics
-
The top 20 publishers
+
Site Usage
+
+ ${usage_nav('Publishers', None)}
+