From: Ross Jones Date: Fri, 02 Nov 2012 10:29:28 +0000 Subject: Changed the icons in the minor nav to simpler icons X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=ea0d616a0a5c80543e05d79814bbd44b800bdead --- Changed the icons in the minor nav to simpler icons --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,11 +1,15 @@ +import re +import csv +import sys import logging import operator -from ckan.lib.base import BaseController, c, render, request, response +import collections +from ckan.lib.base import (BaseController, c, g, render, request, response, abort) import sqlalchemy from sqlalchemy import func, cast, Integer import ckan.model as model -from ga_model import GA_Url, GA_Stat +from ga_model import GA_Url, GA_Stat, GA_ReferralStat log = logging.getLogger('ckanext.ga-report') @@ -30,12 +34,13 @@ def csv(self, month): import csv - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.period_name==month).\ - order_by('GA_Stat.stat_name, GA_Stat.key').all() - - response.headers['Content-disposition'] = 'attachment; filename=dgu_analytics_%s.csv' % (month,) + q = model.Session.query(GA_Stat) + if month != 'all': + q = q.filter(GA_Stat.period_name==month) + entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) writer = csv.writer(response) writer.writerow(["Period", "Statistic", "Key", "Value"]) @@ -53,30 +58,131 @@ c.months = _month_details(GA_Stat) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name=='Totals').\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.key').all() - c.global_totals = [(s.key, s.value) for s in entries ] + c.month_desc = 'all months' + c.month = request.params.get('month', '') + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals') + if c.month: + q = q.filter(GA_Stat.period_name==c.month) + entries = q.order_by('ga_stat.key').all() + + def clean_key(key, val): + if key in ['Average time on site', 'Pages per visit', 'New visits']: + val = "%.2f" % round(float(val), 2) + if key == 'Average time on site': + mins, secs = divmod(float(val), 60) + hours, mins = divmod(mins, 60) + val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) + if key == 'New visits': + val = "%s%%" % val + if key in ['Bounces', 'Total page views', 'Total visits']: + val = int(val) + + return key, val + + c.global_totals = [] + if c.month: + for e in entries: + key, val = clean_key(e.key, e.value) + c.global_totals.append((key, val)) + else: + d = collections.defaultdict(list) + for e in entries: + d[e.key].append(float(e.value)) + for k, v in d.iteritems(): + if k in ['Bounces', 'Total page views', 'Total visits']: + v = sum(v) + else: + v = float(sum(v))/len(v) + key, val = clean_key(k,v) + c.global_totals.append((key, val)) + c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) keys = { - 'Browser versions': 'browsers', - 'Operating Systems versions': 'os', + 'Browser versions': 'browser_versions', + 'Browsers': 'browsers', + 'Operating Systems versions': 'os_versions', + 'Operating Systems': 'os', 'Social sources': 'social_networks', 'Languages': 'languages', 'Country': 'country' } + def shorten_name(name, length=60): + return (name[:length] + '..') if len(name) > 60 else name + + def fill_out_url(url): + import urlparse + return urlparse.urljoin(g.site_url, url) + + c.social_referrer_totals, c.social_referrers = [], [] + q = model.Session.query(GA_ReferralStat) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('ga_referrer.count::int desc') + for entry in q.all(): + c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), + entry.source,entry.count)) + + q = model.Session.query(GA_ReferralStat.url, + func.sum(GA_ReferralStat.count).label('count')) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('count desc').group_by(GA_ReferralStat.url) + for entry in q.all(): + c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', + entry[1])) + + + browser_version_re = re.compile("(.*)\((.*)\)") for k, v in keys.iteritems(): - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name==k).\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.value::int desc').all() - setattr(c, v, [(s.key, s.value) for s in entries ]) - + + def clean_field(key): + if k != 'Browser versions': + return key + m = browser_version_re.match(key) + browser = m.groups()[0].strip() + ver = m.groups()[1] + parts = ver.split('.') + if len(parts) > 1: + if parts[1][0] == '0': + ver = parts[0] + else: + ver = "%s.%s" % (parts[0],parts[1]) + if browser in ['Safari','Android Browser']: # Special case complex version nums + ver = parts[0] + if len(ver) > 2: + ver = "%s%sX" % (ver[0], ver[1]) + + return "%s (%s)" % (browser, ver,) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k) + if c.month: + entries = [] + q = q.filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc') + + d = collections.defaultdict(int) + for e in q.all(): + d[clean_field(e.key)] += int(e.value) + entries = [] + for key, val in d.iteritems(): + entries.append((key,val,)) + entries = sorted(entries, key=operator.itemgetter(1), reverse=True) + + def percent(num, total): + p = 100 * float(num)/float(total) + return "%.2f%%" % round(p, 2) + + # Get the total for each set of values and then set the value as + # a percentage of the total + if k == 'Social sources': + total = sum([x for n,x in c.global_totals if n == 'Total visits']) + else: + total = sum([num for _,num in entries]) + setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) return render('ga_report/site/index.html') @@ -86,6 +192,45 @@ Displays the pageview and visit count for specific publishers based on the datasets associated with the publisher. """ + def csv(self, month): + + c.month = month if not month =='all' else '' + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for publisher,view,visit in self._get_publishers(None): + writer.writerow([publisher.title.encode('utf-8'), + view, + visit, + month]) + + + + def publisher_csv(self, id, month): + + c.month = month if not month =='all' else '' + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + + packages = self._get_packages(c.publisher) + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = \ + str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for package,view,visit in packages: + writer.writerow([package.title.encode('utf-8'), + view, + visit, + month]) + + def index(self): @@ -94,28 +239,85 @@ c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - + c.month = request.params.get('month', '') + c.month_desc = 'all months' + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.top_publishers = self._get_publishers() + + return render('ga_report/publisher/index.html') + + def _get_publishers(self, limit=20): connection = model.Session.connection() q = """ select department_id, sum(pageviews::int) views, sum(visitors::int) visits from ga_url - where department_id <> '' - and not url like '/publisher/%%' - and period_name=%s - group by department_id order by views desc limit 20; - """ - c.top_publishers = [] + where department_id <> ''""" + if c.month: + q = q + """ + and period_name=%s + """ + q = q + """ + group by department_id order by views desc + """ + if limit: + q = q + " limit %s;" % (limit) + + # Add this back (before and period_name =%s) if you want to ignore publisher + # homepage views + # and not url like '/publisher/%%' + + top_publishers = [] res = connection.execute(q, c.month) + for row in res: - c.top_publishers.append((model.Group.get(row[0]), row[1], row[2])) - - return render('ga_report/publisher/index.html') + g = model.Group.get(row[0]) + if g: + top_publishers.append((g, row[1], row[2])) + return top_publishers + + def _get_packages(self, publisher, count=-1): + if count == -1: + count = sys.maxint + + top_packages = [] + q = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')) + if c.month: + q = q.filter(GA_Url.period_name==c.month) + q = q.order_by('ga_url.pageviews::int desc') + + if c.month: + for entry in q[:count]: + p = model.Package.get(entry.url[len('/dataset/'):]) + top_packages.append((p,entry.pageviews,entry.visitors)) + else: + ds = {} + for entry in q.all(): + if len(ds) >= count: + break + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in ds: + ds[p] = {'views':0, 'visits': 0} + ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) + ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in ds.iteritems(): + results.append((k,v['views'],v['visits'])) + + top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + return top_packages def read(self, id): + count = 20 + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') c.top_packages = [] # package, dataset_views in c.top_packages # Get the month details by fetching distinct values and determining the @@ -123,22 +325,23 @@ c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - entry = model.Session.query(GA_Url).\ - filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\ - filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 - - entries = model.Session.query(GA_Url).\ - filter(GA_Url.department_id==c.publisher.name).\ - filter(GA_Url.period_name==c.month).\ - order_by('ga_url.pageviews::int desc')[:20] - for entry in entries: - if entry.url.startswith('/dataset/'): - p = model.Package.get(entry.url[len('/dataset/'):]) - c.top_packages.append((p,entry.pageviews,entry.visitors)) + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all months' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher.name) + if c.month: + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + else: + for e in q.all(): + c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + + c.top_packages = self._get_packages(c.publisher, 20) return render('ga_report/publisher/read.html') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -1,7 +1,7 @@ import os import logging import datetime - +import collections from pylons import config import ga_model @@ -94,7 +94,6 @@ self.get_full_period_name(period_name, period_complete_day), start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) @@ -107,6 +106,33 @@ ga_model.update_publisher_stats(period_name) # about 30 seconds. self.sitewide_stats( period_name ) + + self.update_social_info(period_name, start_date, end_date) + + def update_social_info(self, period_name, start_date, end_date): + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + query = 'ga:hasSocialSourceReferral=~Yes$' + metrics = 'ga:entrances' + sort = '-ga:entrances' + + # Supported query params at + # https://developers.google.com/analytics/devguides/reporting/core/v3/reference + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters=query, + start_date=start_date, + metrics=metrics, + sort=sort, + dimensions="ga:landingPagePath,ga:socialNetwork", + max_results=10000, + end_date=end_date).execute() + data = collections.defaultdict(list) + rows = results.get('rows',[]) + for row in rows: + from ga_model import _normalize_url + data[_normalize_url(row[0])].append( (row[1], int(row[2]),) ) + ga_model.update_social(period_name, data) def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'): @@ -176,12 +202,12 @@ max_results=10000, end_date=end_date).execute() result_data = results.get('rows') - ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]}) - - results = self.service.data().ga().get( - ids='ga:' + self.profile_id, - start_date=start_date, - metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits', + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors', max_results=10000, end_date=end_date).execute() result_data = results.get('rows') @@ -189,7 +215,8 @@ 'Pages per visit': result_data[0][0], 'Bounces': result_data[0][1], 'Average time on site': result_data[0][2], - 'Percent new visits': result_data[0][3], + 'New visits': result_data[0][3], + 'Total visits': result_data[0][4], } ga_model.update_sitewide_stats(period_name, "Totals", data) --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -13,6 +13,8 @@ def make_uuid(): return unicode(uuid.uuid4()) +metadata = MetaData() + class GA_Url(object): @@ -21,20 +23,6 @@ for k,v in kwargs.items(): setattr(self, k, v) -class GA_Stat(object): - - def __init__(self, **kwargs): - for k,v in kwargs.items(): - setattr(self, k, v) - -class GA_Publisher(object): - - def __init__(self, **kwargs): - for k,v in kwargs.items(): - setattr(self, k, v) - - -metadata = MetaData() url_table = Table('ga_url', metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), @@ -47,6 +35,13 @@ ) mapper(GA_Url, url_table) + +class GA_Stat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + stat_table = Table('ga_stat', metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), @@ -56,6 +51,12 @@ Column('value', types.UnicodeText), ) mapper(GA_Stat, stat_table) + +class GA_Publisher(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) pub_table = Table('ga_publisher', metadata, Column('id', types.UnicodeText, primary_key=True, @@ -71,6 +72,24 @@ mapper(GA_Publisher, pub_table) +class GA_ReferralStat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +referrer_table = Table('ga_referrer', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('source', types.UnicodeText), + Column('url', types.UnicodeText), + Column('count', types.Integer), + ) +mapper(GA_ReferralStat, referrer_table) + + + def init_tables(): metadata.create_all(model.meta.engine) @@ -93,8 +112,9 @@ >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices') '/dataset/weekly_fuel_prices' ''' - url = re.sub('https?://(www\.)?data.gov.uk', '', url) - return url + # Deliberately leaving a / + url = url.replace('http:/','') + return '/' + '/'.join(url.split('/')[2:]) def _get_department_id_of_url(url): @@ -167,6 +187,33 @@ model.Session.commit() +def update_social(period_name, data): + # Clean up first. + model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).delete() + + for url,data in data.iteritems(): + for entry in data: + source = entry[0] + count = entry[1] + + item = model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).\ + filter(GA_ReferralStat.source==source).\ + filter(GA_ReferralStat.url==url).first() + if item: + item.count = item.count + count + model.Session.add(item) + else: + # create the row + values = {'id': make_uuid(), + 'period_name': period_name, + 'source': source, + 'url': url, + 'count': count, + } + model.Session.add(GA_ReferralStat(**values)) + model.Session.commit() def update_publisher_stats(period_name): """ --- /dev/null +++ b/ckanext/ga_report/helpers.py @@ -1,1 +1,42 @@ +import logging +import operator +import ckan.lib.base as base +import ckan.model as model +_log = logging.getLogger(__name__) + +def most_popular_datasets(publisher, count=20): + from ckanext.ga_report.ga_model import GA_Url + + if not publisher: + _log.error("No valid publisher passed to 'most_popular_datasets'") + return "" + + datasets = {} + entries = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')).\ + order_by('ga_url.pageviews::int desc').all() + for entry in entries: + if len(datasets) < count: + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in datasets: + datasets[p] = {'views':0, 'visits': 0} + datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) + datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in datasets.iteritems(): + results.append((k,v['views'],v['visits'])) + + results = sorted(results, key=operator.itemgetter(1), reverse=True) + + ctx = { + 'dataset_count': len(datasets), + 'datasets': results, + + 'publisher': publisher + } + + return base.render_snippet('ga_report/publisher/popular.html', **ctx) + --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -2,38 +2,57 @@ import ckan.lib.helpers as h import ckan.plugins as p from ckan.plugins import implements, toolkit -#import gasnippet -#import commands -#import dbutil log = logging.getLogger('ckanext.ga-report') class GAReportPlugin(p.SingletonPlugin): implements(p.IConfigurer, inherit=True) implements(p.IRoutes, inherit=True) + implements(p.ITemplateHelpers, inherit=True) def update_config(self, config): toolkit.add_template_directory(config, 'templates') toolkit.add_public_directory(config, 'public') + def get_helpers(self): + """ + A dictionary of extra helpers that will be available to provide + ga report info to templates. + """ + from ckanext.ga_report.helpers import most_popular_datasets + return { + 'ga_report_installed': lambda: True, + 'most_popular_datasets': most_popular_datasets, + } + def after_map(self, map): map.connect( - '/data/analytics/publisher/', + '/data/site-usage/publisher', controller='ckanext.ga_report.controller:GaPublisherReport', action='index' ) map.connect( - '/data/analytics/publisher/{id}', + '/data/site-usage/publisher_{month}.csv', + controller='ckanext.ga_report.controller:GaPublisherReport', + action='csv' + ) + map.connect( + '/data/site-usage/publisher/{id}_{month}.csv', + controller='ckanext.ga_report.controller:GaPublisherReport', + action='publisher_csv' + ) + map.connect( + '/data/site-usage/publisher/{id}', controller='ckanext.ga_report.controller:GaPublisherReport', action='read' ) map.connect( - '/data/analytics', + '/data/site-usage', controller='ckanext.ga_report.controller:GaReport', action='index' ) map.connect( - '/data/analytics/data_{month}.csv', + '/data/site-usage/data_{month}.csv', controller='ckanext.ga_report.controller:GaReport', action='csv' ) --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -1,1 +1,58 @@ + + + + + + + + + + + + + + +
NameSourceVisits
${name}${source}${count}
+ + + + + + + + + + + + + +
Name% ${title}
${name}${value}
+ + +
+
+ +
+
+ + + + --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -2,22 +2,41 @@ xmlns:i18n="http://genshi.edgewall.org/i18n" xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> + + Publisher Analytics for ${g.site_title}

  • Publishers

    +

    + Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. +

    +

    + Note: this data does not include API calls. +

    +
    • +
  • +

    Download

    +

    + Download as CSV
    +

    • +
    -

    Publisher Analytics

    -

    The top 20 publishers

    +

    Site Usage

    + + ${usage_nav('Publishers', None)} +
    + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -3,22 +3,35 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> - Site analytics + + + Site usage
  • -

    Statistics

    -

    It is possible to export the analytics data as a CSV file, which contains all of the information for ${c.month_desc}

    +

    Site-wide

    +

    + Note: this data does not include API calls and some values have been rounded up to 2 decimal places. Where there are a large number of browser versions they have been grouped together. +

    +
    • +
  • +

    Download

    +

    + Download as CSV
    +

    +
    • -
    -

    Site statistics

    +

    Site Usage

    + ${usage_nav('Site-wide', None)}