From: Ross Jones Date: Fri, 02 Nov 2012 17:50:41 +0000 Subject: New single entry random dataset helper X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=6ca9caae693d4d8ec451fb9a7b61290bc4cd18d6 --- New single entry random dataset helper --- --- a/README.rst +++ b/README.rst @@ -26,17 +26,15 @@ 1. Activate you CKAN python environment and install this extension's software:: $ pyenv/bin/activate - $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report + $ pip install -e git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration:: googleanalytics.id = UA-1010101-1 googleanalytics.account = Account name (i.e. data.gov.uk, see top level item at https://www.google.com/analytics) - googleanalytics.username = googleaccount@gmail.com - googleanalytics.password = googlepassword ga-report.period = monthly - Note that your password will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose. + Note that your credentials will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose. 3. Set up this extension's database tables using a paster command. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file):: --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -66,8 +66,7 @@ And where is: all - data for all time latest - (default) just the 'latest' data - YYYY-MM-DD - just data for all time periods going - back to (and including) this date + YYYY-MM - just data for the specific month """ summary = __doc__.split('\n')[0] usage = __doc__ @@ -96,6 +95,7 @@ elif time_period == 'latest': downloader.latest() else: - since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d') - downloader.since_date(since_date) + # The month to use + for_date = datetime.datetime.strptime(time_period, '%Y-%m') + downloader.specific_month(for_date) --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,10 +1,348 @@ +import re +import csv +import sys import logging -from ckan.lib.base import BaseController, c, render -import report_model +import operator +import collections +from ckan.lib.base import (BaseController, c, g, render, request, response, abort) + +import sqlalchemy +from sqlalchemy import func, cast, Integer +import ckan.model as model +from ga_model import GA_Url, GA_Stat, GA_ReferralStat log = logging.getLogger('ckanext.ga-report') + +def _get_month_name(strdate): + import calendar + from time import strptime + d = strptime(strdate, '%Y-%m') + return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) + + +def _month_details(cls): + months = [] + vals = model.Session.query(cls.period_name).distinct().all() + for m in vals: + months.append( (m[0], _get_month_name(m[0]))) + return sorted(months, key=operator.itemgetter(0), reverse=True) + + class GaReport(BaseController): + + def csv(self, month): + import csv + + q = model.Session.query(GA_Stat) + if month != 'all': + q = q.filter(GA_Stat.period_name==month) + entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() + + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Period", "Statistic", "Key", "Value"]) + + for entry in entries: + writer.writerow([entry.period_name.encode('utf-8'), + entry.stat_name.encode('utf-8'), + entry.key.encode('utf-8'), + entry.value.encode('utf-8')]) + def index(self): - return render('index.html') - + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Stat) + + # Work out which month to show, based on query params of the first item + c.month_desc = 'all months' + c.month = request.params.get('month', '') + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals') + if c.month: + q = q.filter(GA_Stat.period_name==c.month) + entries = q.order_by('ga_stat.key').all() + + def clean_key(key, val): + if key in ['Average time on site', 'Pages per visit', 'New visits']: + val = "%.2f" % round(float(val), 2) + if key == 'Average time on site': + mins, secs = divmod(float(val), 60) + hours, mins = divmod(mins, 60) + val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) + if key == 'New visits': + val = "%s%%" % val + if key in ['Bounces', 'Total page views', 'Total visits']: + val = int(val) + + return key, val + + c.global_totals = [] + if c.month: + for e in entries: + key, val = clean_key(e.key, e.value) + c.global_totals.append((key, val)) + else: + d = collections.defaultdict(list) + for e in entries: + d[e.key].append(float(e.value)) + for k, v in d.iteritems(): + if k in ['Bounces', 'Total page views', 'Total visits']: + v = sum(v) + else: + v = float(sum(v))/len(v) + key, val = clean_key(k,v) + c.global_totals.append((key, val)) + c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) + + keys = { + 'Browser versions': 'browser_versions', + 'Browsers': 'browsers', + 'Operating Systems versions': 'os_versions', + 'Operating Systems': 'os', + 'Social sources': 'social_networks', + 'Languages': 'languages', + 'Country': 'country' + } + + def shorten_name(name, length=60): + return (name[:length] + '..') if len(name) > 60 else name + + def fill_out_url(url): + import urlparse + return urlparse.urljoin(g.site_url, url) + + c.social_referrer_totals, c.social_referrers = [], [] + q = model.Session.query(GA_ReferralStat) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('ga_referrer.count::int desc') + for entry in q.all(): + c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), + entry.source,entry.count)) + + q = model.Session.query(GA_ReferralStat.url, + func.sum(GA_ReferralStat.count).label('count')) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('count desc').group_by(GA_ReferralStat.url) + for entry in q.all(): + c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', + entry[1])) + + + browser_version_re = re.compile("(.*)\((.*)\)") + for k, v in keys.iteritems(): + + def clean_field(key): + if k != 'Browser versions': + return key + m = browser_version_re.match(key) + browser = m.groups()[0].strip() + ver = m.groups()[1] + parts = ver.split('.') + if len(parts) > 1: + if parts[1][0] == '0': + ver = parts[0] + else: + ver = "%s.%s" % (parts[0],parts[1]) + if browser in ['Safari','Android Browser']: # Special case complex version nums + ver = parts[0] + if len(ver) > 2: + ver = "%s%sX" % (ver[0], ver[1]) + + return "%s (%s)" % (browser, ver,) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k) + if c.month: + entries = [] + q = q.filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc') + + d = collections.defaultdict(int) + for e in q.all(): + d[clean_field(e.key)] += int(e.value) + entries = [] + for key, val in d.iteritems(): + entries.append((key,val,)) + entries = sorted(entries, key=operator.itemgetter(1), reverse=True) + + def percent(num, total): + p = 100 * float(num)/float(total) + return "%.2f%%" % round(p, 2) + + # Get the total for each set of values and then set the value as + # a percentage of the total + if k == 'Social sources': + total = sum([x for n,x in c.global_totals if n == 'Total visits']) + else: + total = sum([num for _,num in entries]) + setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) + + return render('ga_report/site/index.html') + + +class GaPublisherReport(BaseController): + """ + Displays the pageview and visit count for specific publishers based on + the datasets associated with the publisher. + """ + def csv(self, month): + + c.month = month if not month =='all' else '' + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for publisher,view,visit in _get_publishers(None): + writer.writerow([publisher.title.encode('utf-8'), + view, + visit, + month]) + + + + def publisher_csv(self, id, month): + + c.month = month if not month =='all' else '' + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + + packages = self._get_packages(c.publisher) + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = \ + str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for package,view,visit in packages: + writer.writerow([package.title.encode('utf-8'), + view, + visit, + month]) + + + + def index(self): + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', '') + c.month_desc = 'all months' + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.top_publishers = _get_publishers() + + return render('ga_report/publisher/index.html') + + + def _get_packages(self, publisher, count=-1): + if count == -1: + count = sys.maxint + + top_packages = [] + q = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')) + if c.month: + q = q.filter(GA_Url.period_name==c.month) + q = q.order_by('ga_url.pageviews::int desc') + + if c.month: + for entry in q[:count]: + p = model.Package.get(entry.url[len('/dataset/'):]) + top_packages.append((p,entry.pageviews,entry.visitors)) + else: + ds = {} + for entry in q.all(): + if len(ds) >= count: + break + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in ds: + ds[p] = {'views':0, 'visits': 0} + ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) + ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in ds.iteritems(): + results.append((k,v['views'],v['visits'])) + + top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + return top_packages + + + def read(self, id): + count = 20 + + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + c.top_packages = [] # package, dataset_views in c.top_packages + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all months' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher.name) + if c.month: + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + else: + for e in q.all(): + c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + + c.top_packages = self._get_packages(c.publisher, 20) + + return render('ga_report/publisher/read.html') + +def _get_publishers(limit=20): + connection = model.Session.connection() + q = """ + select department_id, sum(pageviews::int) views, sum(visitors::int) visits + from ga_url + where department_id <> ''""" + if c.month: + q = q + """ + and period_name=%s + """ + q = q + """ + group by department_id order by views desc + """ + if limit: + q = q + " limit %s;" % (limit) + + # Add this back (before and period_name =%s) if you want to ignore publisher + # homepage views + # and not url like '/publisher/%%' + + top_publishers = [] + res = connection.execute(q, c.month) + + for row in res: + g = model.Group.get(row[0]) + if g: + top_publishers.append((g, row[1], row[2])) + return top_publishers + --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -1,6 +1,7 @@ +import os import logging import datetime - +import collections from pylons import config import ga_model @@ -20,8 +21,17 @@ self.profile_id = profile_id - def all_(self): - self.since_date(datetime.datetime(2010, 1, 1)) + def specific_month(self, date): + import calendar + + first_of_this_month = datetime.datetime(date.year, date.month, 1) + _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month)) + last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month) + periods = ((date.strftime(FORMAT_MONTH), + last_day_of_month, + first_of_this_month, last_of_this_month),) + self.download_and_store(periods) + def latest(self): if self.period == 'monthly': @@ -36,13 +46,13 @@ self.download_and_store(periods) - def since_date(self, since_date): + def for_date(self, for_date): assert isinstance(since_date, datetime.datetime) periods = [] # (period_name, period_complete_day, start_date, end_date) if self.period == 'monthly': first_of_the_months_until_now = [] - year = since_date.year - month = since_date.month + year = for_date.year + month = for_date.month now = datetime.datetime.now() first_of_this_month = datetime.datetime(now.year, now.month, 1) while True: @@ -84,21 +94,53 @@ self.get_full_period_name(period_name, period_complete_day), start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - data = self.download(start_date, end_date) - log.info('Storing Analytics for period "%s"', + data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') + log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) - self.store(period_name, period_complete_day, data) - - - def download(self, start_date, end_date): + self.store(period_name, period_complete_day, data, ) + + data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+') + log.info('Storing Publisher Analytics for period "%s"', + self.get_full_period_name(period_name, period_complete_day)) + self.store(period_name, period_complete_day, data,) + + ga_model.update_publisher_stats(period_name) # about 30 seconds. + self.sitewide_stats( period_name ) + + self.update_social_info(period_name, start_date, end_date) + + def update_social_info(self, period_name, start_date, end_date): + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + query = 'ga:hasSocialSourceReferral=~Yes$' + metrics = 'ga:entrances' + sort = '-ga:entrances' + + # Supported query params at + # https://developers.google.com/analytics/devguides/reporting/core/v3/reference + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters=query, + start_date=start_date, + metrics=metrics, + sort=sort, + dimensions="ga:landingPagePath,ga:socialNetwork", + max_results=10000, + end_date=end_date).execute() + data = collections.defaultdict(list) + rows = results.get('rows',[]) + for row in rows: + from ga_model import _normalize_url + data[_normalize_url(row[0])].append( (row[1], int(row[2]),) ) + ga_model.update_social(period_name, data) + + + def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'): '''Get data from GA for a given time period''' start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') - # url - #query = 'ga:pagePath=~^%s,ga:pagePath=~^%s' % \ - # (PACKAGE_URL, self.resource_url_tag) - query = 'ga:pagePath=~/dataset/[a-z0-9-]+$' - metrics = 'ga:uniquePageviews' + query = 'ga:pagePath=%s$' % path + metrics = 'ga:uniquePageviews, ga:visitors' sort = '-ga:uniquePageviews' # Supported query params at @@ -110,29 +152,184 @@ metrics=metrics, sort=sort, dimensions="ga:pagePath", - end_date=end_date).execute() - - import pprint - pprint.pprint(results) - print 'Total results: %s' % results.get('totalResults') + max_results=10000, + end_date=end_date).execute() + + if os.getenv('DEBUG'): + import pprint + pprint.pprint(results) + print 'Total results: %s' % results.get('totalResults') packages = [] for entry in results.get('rows'): - (loc,size,) = entry - packages.append( ('http:/' + loc,size, '',) ) + (loc,pageviews,visits) = entry + packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack return dict(url=packages) - - def print_results(self, results): - import pprint - pprint.pprint(results) - if results: - print 'Profile: %s' % results.get('profileInfo').get('profileName') - - print 'Total Visits: %s' % results.get('rows', [[-1]])[0][0] - else: - print 'No results found' def store(self, period_name, period_complete_day, data): if 'url' in data: ga_model.update_url_stats(period_name, period_complete_day, data['url']) + def sitewide_stats(self, period_name): + import calendar + year, month = period_name.split('-') + _, last_day_of_month = calendar.monthrange(int(year), int(month)) + + start_date = '%s-01' % period_name + end_date = '%s-%s' % (period_name, last_day_of_month) + print 'Sitewide_stats for %s (%s -> %s)' % (period_name, start_date, end_date) + + funcs = ['_totals_stats', '_social_stats', '_os_stats', + '_locale_stats', '_browser_stats', '_mobile_stats'] + for f in funcs: + print ' + Fetching %s stats' % f.split('_')[1] + getattr(self, f)(start_date, end_date, period_name) + + def _get_results(result_data, f): + data = {} + for result in result_data: + key = f(result) + data[key] = data.get(key,0) + result[1] + return data + + def _totals_stats(self, start_date, end_date, period_name): + """ Fetches distinct totals, total pageviews etc """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors', + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = { + 'Pages per visit': result_data[0][0], + 'Bounces': result_data[0][1], + 'Average time on site': result_data[0][2], + 'New visits': result_data[0][3], + 'Total visits': result_data[0][4], + } + ga_model.update_sitewide_stats(period_name, "Totals", data) + + + def _locale_stats(self, start_date, end_date, period_name): + """ Fetches stats about language and country """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + dimensions="ga:language,ga:country", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Languages", data) + + data = {} + for result in result_data: + data[result[1]] = data.get(result[1], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Country", data) + + + def _social_stats(self, start_date, end_date, period_name): + """ Finds out which social sites people are referred from """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + dimensions="ga:socialNetwork,ga:referralPath", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + twitter_links = [] + data = {} + for result in result_data: + if not result[0] == '(not set)': + data[result[0]] = data.get(result[0], 0) + int(result[2]) + if result[0] == 'Twitter': + twitter_links.append(result[1]) + ga_model.update_sitewide_stats(period_name, "Social sources", data) + + + def _os_stats(self, start_date, end_date, period_name): + """ Operating system stats """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + dimensions="ga:operatingSystem,ga:operatingSystemVersion", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Operating Systems", data) + + data = {} + for result in result_data: + key = "%s (%s)" % (result[0],result[1]) + data[key] = result[2] + ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data) + + + def _browser_stats(self, start_date, end_date, period_name): + """ Information about browsers and browser versions """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + dimensions="ga:browser,ga:browserVersion", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Browsers", data) + + data = {} + for result in result_data: + key = "%s (%s)" % (result[0], result[1]) + data[key] = result[2] + ga_model.update_sitewide_stats(period_name, "Browser versions", data) + + + def _mobile_stats(self, start_date, end_date, period_name): + """ Info about mobile devices """ + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:uniquePageviews', + sort='-ga:uniquePageviews', + dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo", + max_results=10000, + end_date=end_date).execute() + + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Mobile brands", data) + + data = {} + for result in result_data: + data[result[1]] = data.get(result[1], 0) + int(result[2]) + ga_model.update_sitewide_stats(period_name, "Mobile devices", data) + --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -13,6 +13,8 @@ def make_uuid(): return unicode(uuid.uuid4()) +metadata = MetaData() + class GA_Url(object): @@ -21,18 +23,71 @@ for k,v in kwargs.items(): setattr(self, k, v) - -metadata = MetaData() url_table = Table('ga_url', metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), Column('period_name', types.UnicodeText), Column('period_complete_day', types.Integer), - Column('visits', types.Integer), + Column('pageviews', types.UnicodeText), + Column('visitors', types.UnicodeText), Column('url', types.UnicodeText), - Column('next_page', types.UnicodeText), + Column('department_id', types.UnicodeText), ) mapper(GA_Url, url_table) + + +class GA_Stat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +stat_table = Table('ga_stat', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('stat_name', types.UnicodeText), + Column('key', types.UnicodeText), + Column('value', types.UnicodeText), ) +mapper(GA_Stat, stat_table) + + +class GA_Publisher(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +pub_table = Table('ga_publisher', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('publisher_name', types.UnicodeText), + Column('views', types.UnicodeText), + Column('visitors', types.UnicodeText), + Column('toplevel', types.Boolean, default=False), + Column('subpublishercount', types.Integer, default=0), + Column('parent', types.UnicodeText), +) +mapper(GA_Publisher, pub_table) + + +class GA_ReferralStat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +referrer_table = Table('ga_referrer', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('source', types.UnicodeText), + Column('url', types.UnicodeText), + Column('count', types.Integer), + ) +mapper(GA_ReferralStat, referrer_table) + def init_tables(): @@ -57,8 +112,9 @@ >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices') '/dataset/weekly_fuel_prices' ''' - url = re.sub('https?://(www\.)?data.gov.uk', '', url) - return url + # Deliberately leaving a / + url = url.replace('http:/','') + return '/' + '/'.join(url.split('/')[2:]) def _get_department_id_of_url(url): @@ -71,21 +127,51 @@ if dataset: publisher_groups = dataset.get_groups('publisher') if publisher_groups: - return publisher_groups[0].id + return publisher_groups[0].name + else: + publisher_match = re.match('/publisher/([^/]+)(/.*)?', url) + if publisher_match: + return publisher_match.groups()[0] + + +def update_sitewide_stats(period_name, stat_name, data): + for k,v in data.iteritems(): + item = model.Session.query(GA_Stat).\ + filter(GA_Stat.period_name==period_name).\ + filter(GA_Stat.key==k).\ + filter(GA_Stat.stat_name==stat_name).first() + if item: + item.period_name = period_name + item.key = k + item.value = v + model.Session.add(item) + else: + # create the row + values = {'id': make_uuid(), + 'period_name': period_name, + 'key': k, + 'value': v, + 'stat_name': stat_name + } + model.Session.add(GA_Stat(**values)) + model.Session.commit() + def update_url_stats(period_name, period_complete_day, url_data): - table = get_table('ga_url') - for url, views, next_page in url_data: + for url, views, visitors in url_data: url = _normalize_url(url) department_id = _get_department_id_of_url(url) + # see if the row for this url & month is in the table already - item = model.Session.query(GA_Url).filter(GA_Url.period_name==period_name).filter(GA_Url.url==url).first() + item = model.Session.query(GA_Url).\ + filter(GA_Url.period_name==period_name).\ + filter(GA_Url.url==url).first() if item: - item.period_name = period_complete_day=period_complete_day - item.views = views + item.period_name = period_name + item.pageviews = views + item.visitors = visitors item.department_id = department_id - item.next_page = next_page model.Session.add(item) else: # create the row @@ -93,10 +179,119 @@ 'period_name': period_name, 'period_complete_day': period_complete_day, 'url': url, - 'views': views, - 'department_id': department_id, - 'next_page': next_page} - obj = GA_Url(**values) - model.Session.add(obj) + 'pageviews': views, + 'visitors': visitors, + 'department_id': department_id + } + model.Session.add(GA_Url(**values)) model.Session.commit() + +def update_social(period_name, data): + # Clean up first. + model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).delete() + + for url,data in data.iteritems(): + for entry in data: + source = entry[0] + count = entry[1] + + item = model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).\ + filter(GA_ReferralStat.source==source).\ + filter(GA_ReferralStat.url==url).first() + if item: + item.count = item.count + count + model.Session.add(item) + else: + # create the row + values = {'id': make_uuid(), + 'period_name': period_name, + 'source': source, + 'url': url, + 'count': count, + } + model.Session.add(GA_ReferralStat(**values)) + model.Session.commit() + +def update_publisher_stats(period_name): + """ + Updates the publisher stats from the data retrieved for /dataset/* + and /publisher/*. Will run against each dataset and generates the + totals for the entire tree beneath each publisher. + """ + toplevel = get_top_level() + publishers = model.Session.query(model.Group).\ + filter(model.Group.type=='publisher').\ + filter(model.Group.state=='active').all() + for publisher in publishers: + views, visitors, subpub = update_publisher(period_name, publisher, publisher.name) + parent, parents = '', publisher.get_groups('publisher') + if parents: + parent = parents[0].name + item = model.Session.query(GA_Publisher).\ + filter(GA_Publisher.period_name==period_name).\ + filter(GA_Publisher.publisher_name==publisher.name).first() + if item: + item.views = views + item.visitors = visitors + item.publisher_name = publisher.name + item.toplevel = publisher in toplevel + item.subpublishercount = subpub + item.parent = parent + model.Session.add(item) + else: + # create the row + values = {'id': make_uuid(), + 'period_name': period_name, + 'publisher_name': publisher.name, + 'views': views, + 'visitors': visitors, + 'toplevel': publisher in toplevel, + 'subpublishercount': subpub, + 'parent': parent + } + model.Session.add(GA_Publisher(**values)) + model.Session.commit() + + +def update_publisher(period_name, pub, part=''): + views,visitors,subpub = 0, 0, 0 + for publisher in go_down_tree(pub): + subpub = subpub + 1 + items = model.Session.query(GA_Url).\ + filter(GA_Url.period_name==period_name).\ + filter(GA_Url.department_id==publisher.name).all() + for item in items: + views = views + int(item.pageviews) + visitors = visitors + int(item.visitors) + + return views, visitors, (subpub-1) + + +def get_top_level(): + '''Returns the top level publishers.''' + return model.Session.query(model.Group).\ + outerjoin(model.Member, model.Member.table_id == model.Group.id and \ + model.Member.table_name == 'group' and \ + model.Member.state == 'active').\ + filter(model.Member.id==None).\ + filter(model.Group.type=='publisher').\ + order_by(model.Group.name).all() + +def get_children(publisher): + '''Finds child publishers for the given publisher (object). (Not recursive)''' + from ckan.model.group import HIERARCHY_CTE + return model.Session.query(model.Group).\ + from_statement(HIERARCHY_CTE).params(id=publisher.id, type='publisher').\ + all() + +def go_down_tree(publisher): + '''Provided with a publisher object, it walks down the hierarchy and yields each publisher, + including the one you supply.''' + yield publisher + for child in get_children(publisher): + for grandchild in go_down_tree(child): + yield grandchild + --- /dev/null +++ b/ckanext/ga_report/helpers.py @@ -1,1 +1,91 @@ +import logging +import operator +import ckan.lib.base as base +import ckan.model as model +from ckanext.ga_report.ga_model import GA_Url, GA_Publisher +from ckanext.ga_report.controller import _get_publishers +_log = logging.getLogger(__name__) + +def popular_datasets(count=10): + import random + + publisher = None + publishers = _get_publishers(30) + total = len(publishers) + while not publisher or not datasets: + rand = random.randrange(0, total) + publisher = publishers[rand][0] + if not publisher.state == 'active': + publisher = None + continue + datasets = _datasets_for_publisher(publisher, 10)[:count] + + ctx = { + 'datasets': datasets, + 'publisher': publisher + } + return base.render_snippet('ga_report/ga_popular_datasets.html', **ctx) + +def single_popular_dataset(top=20): + import random + + datasets = {} + rand = random.randrange(0, top) + entry = model.Session.query(GA_Url).\ + filter(GA_Url.url.like('/dataset/%')).\ + order_by('ga_url.pageviews::int desc')[rand] + + + dataset = None + while not dataset: + dataset = model.Package.get(entry.url[len('/dataset/'):]) + if dataset and not dataset.state == 'active': + dataset = None + else: + publisher = model.Group.get(entry.department_id) + + ctx = { + 'dataset': dataset, + 'publisher': publisher + } + return base.render_snippet('ga_report/ga_popular_single.html', **ctx) + + +def most_popular_datasets(publisher, count=20): + + if not publisher: + _log.error("No valid publisher passed to 'most_popular_datasets'") + return "" + + results = _datasets_for_publisher(publisher, count) + + ctx = { + 'dataset_count': len(datasets), + 'datasets': results, + + 'publisher': publisher + } + + return base.render_snippet('ga_report/publisher/popular.html', **ctx) + +def _datasets_for_publisher(publisher, count): + datasets = {} + entries = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')).\ + order_by('ga_url.pageviews::int desc').all() + for entry in entries: + if len(datasets) < count: + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in datasets: + datasets[p] = {'views':0, 'visits': 0} + datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) + datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in datasets.iteritems(): + results.append((k,v['views'],v['visits'])) + + return sorted(results, key=operator.itemgetter(1), reverse=True) + --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -1,25 +1,65 @@ import logging import ckan.lib.helpers as h +import ckan.plugins as p from ckan.plugins import implements, toolkit -import gasnippet -import commands -import dbutil + +from ckanext.ga_report.helpers import (most_popular_datasets, + popular_datasets, + single_popular_dataset) log = logging.getLogger('ckanext.ga-report') -class GoogleAnalyticsPlugin(p.SingletonPlugin): +class GAReportPlugin(p.SingletonPlugin): implements(p.IConfigurer, inherit=True) implements(p.IRoutes, inherit=True) + implements(p.ITemplateHelpers, inherit=True) def update_config(self, config): toolkit.add_template_directory(config, 'templates') toolkit.add_public_directory(config, 'public') + def get_helpers(self): + """ + A dictionary of extra helpers that will be available to provide + ga report info to templates. + """ + return { + 'ga_report_installed': lambda: True, + 'popular_datasets': popular_datasets, + 'most_popular_datasets': most_popular_datasets, + 'single_popular_dataset': single_popular_dataset + } + def after_map(self, map): map.connect( - '/data/analytics/index', - controller='ckanext.ga-report.controller:GaReport', + '/data/site-usage/publisher', + controller='ckanext.ga_report.controller:GaPublisherReport', action='index' + ) + map.connect( + '/data/site-usage/publisher_{month}.csv', + controller='ckanext.ga_report.controller:GaPublisherReport', + action='csv' + ) + map.connect( + '/data/site-usage/publisher/{id}_{month}.csv', + controller='ckanext.ga_report.controller:GaPublisherReport', + action='publisher_csv' + ) + map.connect( + '/data/site-usage/publisher/{id}', + controller='ckanext.ga_report.controller:GaPublisherReport', + action='read' + ) + map.connect( + '/data/site-usage', + controller='ckanext.ga_report.controller:GaReport', + action='index' + ) + map.connect( + '/data/site-usage/data_{month}.csv', + controller='ckanext.ga_report.controller:GaReport', + action='csv' ) return map --- a/ckanext/ga_report/report_model.py +++ /dev/null --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/ga_popular_datasets.html @@ -1,1 +1,27 @@ + +
+
+ More popular datasets ยป +
+

Popular datasets

+

${publisher.title}

+
    + +
  • + ${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))} +
    ${h.truncate(dataset.notes, length=80, whole_word=True)}
    +
    • +     +
+ +
+ + + + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/ga_popular_single.html @@ -1,1 +1,23 @@ + + +
+

Featured dataset

+ +

${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))}

${h.truncate(dataset.notes, length=200, whole_word=True)}
+

+
+More popular datasets +All usage data +
+
+ + + + + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -1,1 +1,58 @@ + + + + + + + + + + + + + + +
NameSourceVisits
${name}${source}${count}
+ + + + + + + + + + + + + +
Name% ${title}
${name}${value}
+ + +
+
+ +
+
+ + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -1,1 +1,81 @@ + + + + Publisher Analytics for ${g.site_title} + + +
  • +

    Publishers

    +

    + Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. +

    +

    + Note: this data does not include API calls. +

    +
    • +
  • +

    Download

    +

    + Download as CSV
    +

    +
    • +     + + +
    + +

    Site Usage

    + + ${usage_nav('Publishers', None)} + + +
    +
    + + +
    +
    + + + + + + + + + + + + + + +
    PublisherDataset ViewsVisits
    ${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport', action='read', id=publisher.name))} + ${views}${visits}
    + + +
    + + + + + + + + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/publisher/popular.html @@ -1,1 +1,25 @@ + + +

    We do not currently have analytics data for ${publisher.title}

    +     + +
    +
      + +
    • + ${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))} +
      • +       +
    +

    ${h.link_to("More analytics for " + publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=publisher.name))}

    +
    +     + + + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/publisher/read.html @@ -1,1 +1,71 @@ + + + + Analytics for ${g.site_title} + + +
  • +

    Publishers

    +

    + Dataset views records the number of times a specific dataset page has been viewed for ${c.publisher.title}. Visits records the number of unique site visits. +

    +

    + Note: this data does not include API calls. +

    +
    • + +
  • +

    Download

    +

    + Download as CSV
    +

    +
    • +     + + +
    +

    Site Usage

    + + ${usage_nav(c.publisher.title, c.publisher)} + +
    +
    + + +
    +
    + + + + + + + + + + + + + + +
    DatasetViewsVisits
    ${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} + ${views}${visits}
    + + +
    + + + + + + + --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -1,1 +1,142 @@ + + + + Site usage + + +
  • +

    Site-wide

    +

    + Note: this data does not include API calls and some values have been rounded up to 2 decimal places. Where there are a large number of browser versions they have been grouped together. +

    +
    • +
  • +

    Download

    +

    + Download as CSV
    +

    +
    • + +     + +
    +

    Site Usage

    + ${usage_nav('Site-wide', None)} + +
    +
    + + +
    +
    + +
    + +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + ${stat_table(c.browser_versions)} +
    +
    + ${stat_table(c.browsers)} +
    +
    + ${stat_table(c.os)} +
    +
    + ${stat_table(c.os_versions)} +
    +
    +

    Number of visits to urls referred from social networks

    + ${social_table(c.social_referrer_totals)} +
    +
    +

    Percentage of visits referred from these social networks

    + + ${stat_table(c.social_networks, 'Visits')} +
    +
    + ${stat_table(c.languages)} +
    +
    + ${stat_table(c.country)} +
    + + +
    +
    + + + +
    + + + + + + + + + + + --- /dev/null +++ b/ckanext/ga_report/tests/test_api.py @@ -1,1 +1,38 @@ +import os +import datetime +from nose.tools import assert_equal +from ckanext.ga_report.download_analytics import DownloadAnalytics +from ckanext.ga_report.ga_auth import (init_service, get_profile_id) +from ckanext.ga_report.ga_model import init_tables +class TestAPI: + + @classmethod + def setup_class(cls): + if not os.path.exists("token.dat") or not os.path.exists("credentials.json"): + print '*' * 60 + print "Tests may not run without first having run the auth process" + print '*' * 60 + init_tables() + + @classmethod + def teardown_class(cls): + pass + + def test_latest(self): + svc = init_service("token.dat", "credentials.json") + try: + downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) + downloader.latest() + except Exception as e: + assert False, e + + + def test_since(self): + svc = init_service("token.dat", "credentials.json") + downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) + try: + downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30)) + except Exception as e: + assert False, e + --- a/setup.py +++ b/setup.py @@ -27,11 +27,12 @@ """ [ckan.plugins] # Add plugins here, eg - ga-report=ckanext.ga_report.plugin:GaReportPlugin + ga-report=ckanext.ga_report.plugin:GAReportPlugin [paste.paster_command] loadanalytics = ckanext.ga_report.command:LoadAnalytics initdb = ckanext.ga_report.command:InitDB + getauthtoken = ckanext.ga_report.command:GetAuthToken """, )