From: Ross Jones
Date: Thu, 01 Nov 2012 10:44:40 +0000
Subject: Made sure the csv is served as an attachment
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=6c3ecf0aa653f5d746401615a64232e2ad6eba87
---
Made sure the csv is served as an attachment
---
--- a/README.rst
+++ b/README.rst
@@ -26,7 +26,7 @@
1. Activate you CKAN python environment and install this extension's software::
$ pyenv/bin/activate
- $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report
+ $ pip install -e git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report
2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration::
--- a/ckanext/ga_report/command.py
+++ b/ckanext/ga_report/command.py
@@ -66,8 +66,7 @@
And where is:
all - data for all time
latest - (default) just the 'latest' data
- YYYY-MM-DD - just data for all time periods going
- back to (and including) this date
+ YYYY-MM - just data for the specific month
"""
summary = __doc__.split('\n')[0]
usage = __doc__
@@ -96,6 +95,7 @@
elif time_period == 'latest':
downloader.latest()
else:
- since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
- downloader.since_date(since_date)
+ # The month to use
+ for_date = datetime.datetime.strptime(time_period, '%Y-%m')
+ downloader.specific_month(for_date)
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -1,10 +1,311 @@
+import re
+import csv
+import sys
import logging
-from ckan.lib.base import BaseController, c, render
-import report_model
+import operator
+import collections
+from ckan.lib.base import BaseController, c, render, request, response, abort
+
+import sqlalchemy
+from sqlalchemy import func, cast, Integer
+import ckan.model as model
+from ga_model import GA_Url, GA_Stat
log = logging.getLogger('ckanext.ga-report')
+
+def _get_month_name(strdate):
+ import calendar
+ from time import strptime
+ d = strptime(strdate, '%Y-%m')
+ return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)
+
+
+def _month_details(cls):
+ months = []
+ vals = model.Session.query(cls.period_name).distinct().all()
+ for m in vals:
+ months.append( (m[0], _get_month_name(m[0])))
+ return sorted(months, key=operator.itemgetter(0), reverse=True)
+
+
class GaReport(BaseController):
+
+ def csv(self, month):
+ import csv
+
+ q = model.Session.query(GA_Stat)
+ if month != 'all':
+ q = q.filter(GA_Stat.period_name==month)
+ entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all()
+
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,))
+
+ writer = csv.writer(response)
+ writer.writerow(["Period", "Statistic", "Key", "Value"])
+
+ for entry in entries:
+ writer.writerow([entry.period_name.encode('utf-8'),
+ entry.stat_name.encode('utf-8'),
+ entry.key.encode('utf-8'),
+ entry.value.encode('utf-8')])
+
def index(self):
- return render('index.html')
-
+
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Stat)
+
+ # Work out which month to show, based on query params of the first item
+ c.month_desc = 'all time'
+ c.month = request.params.get('month', '')
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name=='Totals')
+ if c.month:
+ q = q.filter(GA_Stat.period_name==c.month)
+ entries = q.order_by('ga_stat.key').all()
+
+ def clean_key(key, val):
+ if key in ['Average time on site', 'Pages per visit', 'Percent new visits']:
+ val = "%.2f" % round(float(val), 2)
+ if key == 'Average time on site':
+ mins, secs = divmod(float(val), 60)
+ hours, mins = divmod(mins, 60)
+ val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
+ key = '%s *' % key
+ if key in ['Bounces', 'Total pageviews']:
+ val = int(val)
+ return key, val
+
+ c.global_totals = []
+ if c.month:
+ for e in entries:
+ key, val = clean_key(e.key, e.value)
+ c.global_totals.append((key, val))
+ else:
+ d = collections.defaultdict(list)
+ for e in entries:
+ d[e.key].append(float(e.value))
+ for k, v in d.iteritems():
+ if k in ['Bounces', 'Total pageviews']:
+ v = sum(v)
+ else:
+ v = float(sum(v))/len(v)
+ key, val = clean_key(k,v)
+ c.global_totals.append((key, val))
+ c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0))
+
+ keys = {
+ 'Browser versions': 'browsers',
+ 'Operating Systems versions': 'os',
+ 'Social sources': 'social_networks',
+ 'Languages': 'languages',
+ 'Country': 'country'
+ }
+
+ browser_version_re = re.compile("(.*)\((.*)\)")
+ for k, v in keys.iteritems():
+
+ def clean_field(key):
+ if k != 'Browser versions':
+ return key
+ m = browser_version_re.match(key)
+ browser = m.groups()[0].strip()
+ ver = m.groups()[1]
+ parts = ver.split('.')
+ if len(parts) > 1:
+ if parts[1][0] == '0':
+ ver = parts[0]
+ else:
+ ver = "%s.%s" % (parts[0],parts[1])
+ if browser in ['Safari','Android Browser']: # Special case complex version nums
+ ver = parts[0]
+ if len(ver) > 2:
+ ver = "%s%sX" % (ver[0], ver[1])
+
+ return "%s (%s)" % (browser, ver,)
+
+ q = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name==k)
+ if c.month:
+ entries = []
+ q = q.filter(GA_Stat.period_name==c.month).\
+ order_by('ga_stat.value::int desc')
+
+ d = collections.defaultdict(int)
+ for e in q.all():
+ d[clean_field(e.key)] += int(e.value)
+ entries = []
+ for key, val in d.iteritems():
+ entries.append((key,val,))
+ entries = sorted(entries, key=operator.itemgetter(1), reverse=True)
+
+ setattr(c, v, [(k,v) for k,v in entries ])
+
+
+
+ return render('ga_report/site/index.html')
+
+
+class GaPublisherReport(BaseController):
+ """
+ Displays the pageview and visit count for specific publishers based on
+ the datasets associated with the publisher.
+ """
+ def csv(self, month):
+
+ c.month = month if not month =='all' else ''
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,))
+
+ writer = csv.writer(response)
+ writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
+
+ for publisher,view,visit in self._get_publishers(None):
+ writer.writerow([publisher.title.encode('utf-8'),
+ view,
+ visit,
+ month])
+
+
+
+ def publisher_csv(self, id, month):
+
+ c.month = month if not month =='all' else ''
+ c.publisher = model.Group.get(id)
+ if not c.publisher:
+ abort(404, 'A publisher with that name could not be found')
+
+ packages = self._get_packages(c.publisher)
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+ response.headers['Content-Disposition'] = \
+ str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,))
+
+ writer = csv.writer(response)
+ writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
+
+ for package,view,visit in packages:
+ writer.writerow([package.title.encode('utf-8'),
+ view,
+ visit,
+ month])
+
+
+
+ def index(self):
+
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Url)
+
+ # Work out which month to show, based on query params of the first item
+ c.month = request.params.get('month', '')
+ c.month_desc = 'all time'
+ if c.month:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ c.top_publishers = self._get_publishers()
+
+ return render('ga_report/publisher/index.html')
+
+ def _get_publishers(self, limit=20):
+ connection = model.Session.connection()
+ q = """
+ select department_id, sum(pageviews::int) views, sum(visitors::int) visits
+ from ga_url
+ where department_id <> ''"""
+ if c.month:
+ q = q + """
+ and period_name=%s
+ """
+ q = q + """
+ group by department_id order by views desc
+ """
+ if limit:
+ q = q + " limit %s;" % (limit)
+
+ # Add this back (before and period_name =%s) if you want to ignore publisher
+ # homepage views
+ # and not url like '/publisher/%%'
+
+ top_publishers = []
+ res = connection.execute(q, c.month)
+
+ for row in res:
+ g = model.Group.get(row[0])
+ if g:
+ top_publishers.append((g, row[1], row[2]))
+ return top_publishers
+
+ def _get_packages(self, publisher, count=-1):
+ if count == -1:
+ count = sys.maxint
+
+ top_packages = []
+ q = model.Session.query(GA_Url).\
+ filter(GA_Url.department_id==publisher.name).\
+ filter(GA_Url.url.like('/dataset/%'))
+ if c.month:
+ q = q.filter(GA_Url.period_name==c.month)
+ q = q.order_by('ga_url.pageviews::int desc')
+
+ if c.month:
+ for entry in q[:count]:
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ top_packages.append((p,entry.pageviews,entry.visitors))
+ else:
+ ds = {}
+ for entry in q.all():
+ if len(ds) >= count:
+ break
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in ds:
+ ds[p] = {'views':0, 'visits': 0}
+ ds[p]['views'] = ds[p]['views'] + int(entry.pageviews)
+ ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors)
+
+ results = []
+ for k, v in ds.iteritems():
+ results.append((k,v['views'],v['visits']))
+
+ top_packages = sorted(results, key=operator.itemgetter(1), reverse=True)
+ return top_packages
+
+
+ def read(self, id):
+ count = 20
+
+ c.publisher = model.Group.get(id)
+ if not c.publisher:
+ abort(404, 'A publisher with that name could not be found')
+ c.top_packages = [] # package, dataset_views in c.top_packages
+
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Url)
+
+ # Work out which month to show, based on query params of the first item
+ c.month = request.params.get('month', '')
+ if not c.month:
+ c.month_desc = 'all time'
+ else:
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ c.publisher_page_views = 0
+ q = model.Session.query(GA_Url).\
+ filter(GA_Url.url=='/publisher/%s' % c.publisher.name)
+ if c.month:
+ entry = q.filter(GA_Url.period_name==c.month).first()
+ c.publisher_page_views = entry.pageviews if entry else 0
+ else:
+ for e in q.all():
+ c.publisher_page_views = c.publisher_page_views + int(e.pageviews)
+
+ c.top_packages = self._get_packages(c.publisher, 20)
+
+ return render('ga_report/publisher/read.html')
+
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,3 +1,4 @@
+import os
import logging
import datetime
@@ -20,8 +21,17 @@
self.profile_id = profile_id
- def all_(self):
- self.since_date(datetime.datetime(2010, 1, 1))
+ def specific_month(self, date):
+ import calendar
+
+ first_of_this_month = datetime.datetime(date.year, date.month, 1)
+ _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month))
+ last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month)
+ periods = ((date.strftime(FORMAT_MONTH),
+ last_day_of_month,
+ first_of_this_month, last_of_this_month),)
+ self.download_and_store(periods)
+
def latest(self):
if self.period == 'monthly':
@@ -36,13 +46,13 @@
self.download_and_store(periods)
- def since_date(self, since_date):
+ def for_date(self, for_date):
assert isinstance(since_date, datetime.datetime)
periods = [] # (period_name, period_complete_day, start_date, end_date)
if self.period == 'monthly':
first_of_the_months_until_now = []
- year = since_date.year
- month = since_date.month
+ year = for_date.year
+ month = for_date.month
now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1)
while True:
@@ -84,18 +94,27 @@
self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d'))
- data = self.download(start_date, end_date)
- log.info('Storing Analytics for period "%s"',
+
+ data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
+ log.info('Storing Dataset Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day))
- self.store(period_name, period_complete_day, data)
-
-
- def download(self, start_date, end_date):
+ self.store(period_name, period_complete_day, data, )
+
+ data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
+ log.info('Storing Publisher Analytics for period "%s"',
+ self.get_full_period_name(period_name, period_complete_day))
+ self.store(period_name, period_complete_day, data,)
+
+ ga_model.update_publisher_stats(period_name) # about 30 seconds.
+ self.sitewide_stats( period_name )
+
+
+ def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
'''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
- query = 'ga:pagePath=~/dataset/[a-z0-9-]+$'
- metrics = 'ga:uniquePageviews'
+ query = 'ga:pagePath=%s$' % path
+ metrics = 'ga:uniquePageviews, ga:visitors'
sort = '-ga:uniquePageviews'
# Supported query params at
@@ -110,18 +129,180 @@
max_results=10000,
end_date=end_date).execute()
-
- import pprint
- pprint.pprint(results)
- print 'Total results: %s' % results.get('totalResults')
+ if os.getenv('DEBUG'):
+ import pprint
+ pprint.pprint(results)
+ print 'Total results: %s' % results.get('totalResults')
packages = []
for entry in results.get('rows'):
- (loc,size,) = entry
- packages.append( ('http:/' + loc,size, '',) ) # Temporary hack
+ (loc,pageviews,visits) = entry
+ packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
return dict(url=packages)
def store(self, period_name, period_complete_day, data):
if 'url' in data:
ga_model.update_url_stats(period_name, period_complete_day, data['url'])
+ def sitewide_stats(self, period_name):
+ import calendar
+ year, month = period_name.split('-')
+ _, last_day_of_month = calendar.monthrange(int(year), int(month))
+
+ start_date = '%s-01' % period_name
+ end_date = '%s-%s' % (period_name, last_day_of_month)
+ print 'Sitewide_stats for %s (%s -> %s)' % (period_name, start_date, end_date)
+
+ funcs = ['_totals_stats', '_social_stats', '_os_stats',
+ '_locale_stats', '_browser_stats', '_mobile_stats']
+ for f in funcs:
+ print ' + Fetching %s stats' % f.split('_')[1]
+ getattr(self, f)(start_date, end_date, period_name)
+
+ def _get_results(result_data, f):
+ data = {}
+ for result in result_data:
+ key = f(result)
+ data[key] = data.get(key,0) + result[1]
+ return data
+
+ def _totals_stats(self, start_date, end_date, period_name):
+ """ Fetches distinct totals, total pageviews etc """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]})
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits',
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {
+ 'Pages per visit': result_data[0][0],
+ 'Bounces': result_data[0][1],
+ 'Average time on site': result_data[0][2],
+ 'Percent new visits': result_data[0][3],
+ }
+ ga_model.update_sitewide_stats(period_name, "Totals", data)
+
+
+ def _locale_stats(self, start_date, end_date, period_name):
+ """ Fetches stats about language and country """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:language,ga:country",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Languages", data)
+
+ data = {}
+ for result in result_data:
+ data[result[1]] = data.get(result[1], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Country", data)
+
+
+ def _social_stats(self, start_date, end_date, period_name):
+ """ Finds out which social sites people are referred from """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:socialNetwork,ga:referralPath",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ twitter_links = []
+ data = {}
+ for result in result_data:
+ if not result[0] == '(not set)':
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ if result[0] == 'Twitter':
+ twitter_links.append(result[1])
+ ga_model.update_sitewide_stats(period_name, "Social sources", data)
+
+
+ def _os_stats(self, start_date, end_date, period_name):
+ """ Operating system stats """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:operatingSystem,ga:operatingSystemVersion",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Operating Systems", data)
+
+ data = {}
+ for result in result_data:
+ key = "%s (%s)" % (result[0],result[1])
+ data[key] = result[2]
+ ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data)
+
+
+ def _browser_stats(self, start_date, end_date, period_name):
+ """ Information about browsers and browser versions """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:browser,ga:browserVersion",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Browsers", data)
+
+ data = {}
+ for result in result_data:
+ key = "%s (%s)" % (result[0], result[1])
+ data[key] = result[2]
+ ga_model.update_sitewide_stats(period_name, "Browser versions", data)
+
+
+ def _mobile_stats(self, start_date, end_date, period_name):
+ """ Info about mobile devices """
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",
+ max_results=10000,
+ end_date=end_date).execute()
+
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Mobile brands", data)
+
+ data = {}
+ for result in result_data:
+ data[result[1]] = data.get(result[1], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Mobile devices", data)
+
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -16,6 +16,18 @@
class GA_Url(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+class GA_Stat(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+class GA_Publisher(object):
def __init__(self, **kwargs):
for k,v in kwargs.items():
@@ -28,12 +40,35 @@
default=make_uuid),
Column('period_name', types.UnicodeText),
Column('period_complete_day', types.Integer),
- Column('metric', types.UnicodeText),
- Column('value', types.UnicodeText),
+ Column('pageviews', types.UnicodeText),
+ Column('visitors', types.UnicodeText),
Column('url', types.UnicodeText),
Column('department_id', types.UnicodeText),
)
mapper(GA_Url, url_table)
+
+stat_table = Table('ga_stat', metadata,
+ Column('id', types.UnicodeText, primary_key=True,
+ default=make_uuid),
+ Column('period_name', types.UnicodeText),
+ Column('stat_name', types.UnicodeText),
+ Column('key', types.UnicodeText),
+ Column('value', types.UnicodeText), )
+mapper(GA_Stat, stat_table)
+
+
+pub_table = Table('ga_publisher', metadata,
+ Column('id', types.UnicodeText, primary_key=True,
+ default=make_uuid),
+ Column('period_name', types.UnicodeText),
+ Column('publisher_name', types.UnicodeText),
+ Column('views', types.UnicodeText),
+ Column('visitors', types.UnicodeText),
+ Column('toplevel', types.Boolean, default=False),
+ Column('subpublishercount', types.Integer, default=0),
+ Column('parent', types.UnicodeText),
+)
+mapper(GA_Publisher, pub_table)
def init_tables():
@@ -73,22 +108,49 @@
publisher_groups = dataset.get_groups('publisher')
if publisher_groups:
return publisher_groups[0].name
+ else:
+ publisher_match = re.match('/publisher/([^/]+)(/.*)?', url)
+ if publisher_match:
+ return publisher_match.groups()[0]
+
+
+def update_sitewide_stats(period_name, stat_name, data):
+ for k,v in data.iteritems():
+ item = model.Session.query(GA_Stat).\
+ filter(GA_Stat.period_name==period_name).\
+ filter(GA_Stat.key==k).\
+ filter(GA_Stat.stat_name==stat_name).first()
+ if item:
+ item.period_name = period_name
+ item.key = k
+ item.value = v
+ model.Session.add(item)
+ else:
+ # create the row
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'key': k,
+ 'value': v,
+ 'stat_name': stat_name
+ }
+ model.Session.add(GA_Stat(**values))
+ model.Session.commit()
+
def update_url_stats(period_name, period_complete_day, url_data):
- table = get_table('ga_url')
- for url, views, next_page in url_data:
+ for url, views, visitors in url_data:
url = _normalize_url(url)
department_id = _get_department_id_of_url(url)
# see if the row for this url & month is in the table already
item = model.Session.query(GA_Url).\
filter(GA_Url.period_name==period_name).\
- filter(GA_Url.url==url).\
- filter(GA_Url.metric == 'Total views').first()
+ filter(GA_Url.url==url).first()
if item:
- item.period_name = period_complete_day = period_complete_day
- item.value = views
+ item.period_name = period_name
+ item.pageviews = views
+ item.visitors = visitors
item.department_id = department_id
model.Session.add(item)
else:
@@ -97,10 +159,92 @@
'period_name': period_name,
'period_complete_day': period_complete_day,
'url': url,
- 'value': views,
- 'metric': 'Total views',
+ 'pageviews': views,
+ 'visitors': visitors,
'department_id': department_id
}
model.Session.add(GA_Url(**values))
model.Session.commit()
+
+
+def update_publisher_stats(period_name):
+ """
+ Updates the publisher stats from the data retrieved for /dataset/*
+ and /publisher/*. Will run against each dataset and generates the
+ totals for the entire tree beneath each publisher.
+ """
+ toplevel = get_top_level()
+ publishers = model.Session.query(model.Group).\
+ filter(model.Group.type=='publisher').\
+ filter(model.Group.state=='active').all()
+ for publisher in publishers:
+ views, visitors, subpub = update_publisher(period_name, publisher, publisher.name)
+ parent, parents = '', publisher.get_groups('publisher')
+ if parents:
+ parent = parents[0].name
+ item = model.Session.query(GA_Publisher).\
+ filter(GA_Publisher.period_name==period_name).\
+ filter(GA_Publisher.publisher_name==publisher.name).first()
+ if item:
+ item.views = views
+ item.visitors = visitors
+ item.publisher_name = publisher.name
+ item.toplevel = publisher in toplevel
+ item.subpublishercount = subpub
+ item.parent = parent
+ model.Session.add(item)
+ else:
+ # create the row
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'publisher_name': publisher.name,
+ 'views': views,
+ 'visitors': visitors,
+ 'toplevel': publisher in toplevel,
+ 'subpublishercount': subpub,
+ 'parent': parent
+ }
+ model.Session.add(GA_Publisher(**values))
+ model.Session.commit()
+
+
+def update_publisher(period_name, pub, part=''):
+ views,visitors,subpub = 0, 0, 0
+ for publisher in go_down_tree(pub):
+ subpub = subpub + 1
+ items = model.Session.query(GA_Url).\
+ filter(GA_Url.period_name==period_name).\
+ filter(GA_Url.department_id==publisher.name).all()
+ for item in items:
+ views = views + int(item.pageviews)
+ visitors = visitors + int(item.visitors)
+
+ return views, visitors, (subpub-1)
+
+
+def get_top_level():
+ '''Returns the top level publishers.'''
+ return model.Session.query(model.Group).\
+ outerjoin(model.Member, model.Member.table_id == model.Group.id and \
+ model.Member.table_name == 'group' and \
+ model.Member.state == 'active').\
+ filter(model.Member.id==None).\
+ filter(model.Group.type=='publisher').\
+ order_by(model.Group.name).all()
+
+def get_children(publisher):
+ '''Finds child publishers for the given publisher (object). (Not recursive)'''
+ from ckan.model.group import HIERARCHY_CTE
+ return model.Session.query(model.Group).\
+ from_statement(HIERARCHY_CTE).params(id=publisher.id, type='publisher').\
+ all()
+
+def go_down_tree(publisher):
+ '''Provided with a publisher object, it walks down the hierarchy and yields each publisher,
+ including the one you supply.'''
+ yield publisher
+ for child in get_children(publisher):
+ for grandchild in go_down_tree(child):
+ yield grandchild
+
--- /dev/null
+++ b/ckanext/ga_report/helpers.py
@@ -1,1 +1,42 @@
+import logging
+import operator
+import ckan.lib.base as base
+import ckan.model as model
+_log = logging.getLogger(__name__)
+
+def most_popular_datasets(publisher, count=20):
+ from ckanext.ga_report.ga_model import GA_Url
+
+ if not publisher:
+ _log.error("No valid publisher passed to 'most_popular_datasets'")
+ return ""
+
+ datasets = {}
+ entries = model.Session.query(GA_Url).\
+ filter(GA_Url.department_id==publisher.name).\
+ filter(GA_Url.url.like('/dataset/%')).\
+ order_by('ga_url.pageviews::int desc').all()
+ for entry in entries:
+ if len(datasets) < count:
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ if not p in datasets:
+ datasets[p] = {'views':0, 'visits': 0}
+ datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
+ datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
+
+ results = []
+ for k, v in datasets.iteritems():
+ results.append((k,v['views'],v['visits']))
+
+ results = sorted(results, key=operator.itemgetter(1), reverse=True)
+
+ ctx = {
+ 'dataset_count': len(datasets),
+ 'datasets': results,
+
+ 'publisher': publisher
+ }
+
+ return base.render_snippet('ga_report/publisher/popular.html', **ctx)
+
--- a/ckanext/ga_report/plugin.py
+++ b/ckanext/ga_report/plugin.py
@@ -1,25 +1,60 @@
import logging
import ckan.lib.helpers as h
+import ckan.plugins as p
from ckan.plugins import implements, toolkit
-import gasnippet
-import commands
-import dbutil
log = logging.getLogger('ckanext.ga-report')
-class GoogleAnalyticsPlugin(p.SingletonPlugin):
+class GAReportPlugin(p.SingletonPlugin):
implements(p.IConfigurer, inherit=True)
implements(p.IRoutes, inherit=True)
+ implements(p.ITemplateHelpers, inherit=True)
def update_config(self, config):
toolkit.add_template_directory(config, 'templates')
toolkit.add_public_directory(config, 'public')
+ def get_helpers(self):
+ """
+ A dictionary of extra helpers that will be available to provide
+ ga report info to templates.
+ """
+ from ckanext.ga_report.helpers import most_popular_datasets
+ return {
+ 'ga_report_installed': lambda: True,
+ 'most_popular_datasets': most_popular_datasets,
+ }
+
def after_map(self, map):
map.connect(
- '/data/analytics/index',
- controller='ckanext.ga-report.controller:GaReport',
+ '/data/site-usage/publisher',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
action='index'
+ )
+ map.connect(
+ '/data/site-usage/publisher_{month}.csv',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
+ action='csv'
+ )
+ map.connect(
+ '/data/site-usage/publisher/{id}_{month}.csv',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
+ action='publisher_csv'
+ )
+ map.connect(
+ '/data/site-usage/publisher/{id}',
+ controller='ckanext.ga_report.controller:GaPublisherReport',
+ action='read'
+ )
+ map.connect(
+ '/data/site-usage',
+ controller='ckanext.ga_report.controller:GaReport',
+ action='index'
+ )
+ map.connect(
+ '/data/site-usage/data_{month}.csv',
+ controller='ckanext.ga_report.controller:GaReport',
+ action='csv'
)
return map
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/ga_util.html
@@ -1,1 +1,37 @@
+
+
+
+
+
+
+
+
+
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -1,1 +1,80 @@
+
+
+
+ Publisher Analytics for ${g.site_title}
+
+
+
+ Publishers
+
+ Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits.
+
+
+ Note: this data does not include API calls.
+
+
+
+ Download
+
+ Download as CSV
+
+
+
+
+
+
+
Site Usage
+
+ ${usage_nav('Publishers', None)}
+
+
+
+
+
+
+ Publisher |
+ Dataset Views |
+ Visits |
+
+
+
+ ${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport', action='read', id=publisher.name))}
+ |
+ ${views} |
+ ${visits} |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/publisher/popular.html
@@ -1,1 +1,25 @@
+
+
+ We do not currently have analytics data for ${publisher.title}
+
+
+
+
+
+ -
+ ${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))}
+
+
+
+
${h.link_to("More analytics for " + publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=publisher.name))}
+
+
+
+
+
+
+
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -1,1 +1,71 @@
+
+
+
+ Analytics for ${g.site_title}
+
+
+
+ Publishers
+
+ Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits.
+
+
+ Note: this data does not include API calls.
+
+
+
+
+ Download
+
+ Download as CSV
+
+
+
+
+
+
+
Site Usage
+
+ ${usage_nav(c.publisher.title, c.publisher)}
+
+
+
+
+
+ Dataset |
+ Views |
+ Visits |
+
+
+
+ ${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}
+ |
+ ${views} |
+ ${visits} |
+
+
+
+
+
+
+
+
+
+
+
+
+
--- /dev/null
+++ b/ckanext/ga_report/templates/ga_report/site/index.html
@@ -1,1 +1,161 @@
+
+
+
+ Site usage
+
+
+
+ Site-wide
+
+ Note: this data does not include API calls and some values have been rounded up to 2 decimal places. Where there are a large number of browser versions they have been grouped together.
+
+
+
+ Download
+
+ Download as CSV
+
+
+
+
+
+
+
Site Usage
+
+ ${usage_nav('Site-wide', None)}
+
+
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+ Name |
+ Value |
+
+
+
+ ${name} |
+ ${value} |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
--- /dev/null
+++ b/ckanext/ga_report/tests/test_api.py
@@ -1,1 +1,38 @@
+import os
+import datetime
+from nose.tools import assert_equal
+from ckanext.ga_report.download_analytics import DownloadAnalytics
+from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
+from ckanext.ga_report.ga_model import init_tables
+class TestAPI:
+
+ @classmethod
+ def setup_class(cls):
+ if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
+ print '*' * 60
+ print "Tests may not run without first having run the auth process"
+ print '*' * 60
+ init_tables()
+
+ @classmethod
+ def teardown_class(cls):
+ pass
+
+ def test_latest(self):
+ svc = init_service("token.dat", "credentials.json")
+ try:
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+ downloader.latest()
+ except Exception as e:
+ assert False, e
+
+
+ def test_since(self):
+ svc = init_service("token.dat", "credentials.json")
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+ try:
+ downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30))
+ except Exception as e:
+ assert False, e
+
--- a/setup.py
+++ b/setup.py
@@ -27,11 +27,12 @@
"""
[ckan.plugins]
# Add plugins here, eg
- ga-report=ckanext.ga_report.plugin:GaReportPlugin
+ ga-report=ckanext.ga_report.plugin:GAReportPlugin
[paste.paster_command]
loadanalytics = ckanext.ga_report.command:LoadAnalytics
initdb = ckanext.ga_report.command:InitDB
+ getauthtoken = ckanext.ga_report.command:GetAuthToken
""",
)