From: Ross Jones Date: Thu, 01 Nov 2012 16:04:16 +0000 Subject: Adding new social network referral information X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=4edba2a7929e3d5505cbed28c7393264cd35f394 --- Adding new social network referral information --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -4,12 +4,12 @@ import logging import operator import collections -from ckan.lib.base import BaseController, c, render, request, response, abort +from ckan.lib.base import (BaseController, c, g, render, request, response, abort) import sqlalchemy from sqlalchemy import func, cast, Integer import ckan.model as model -from ga_model import GA_Url, GA_Stat +from ga_model import GA_Url, GA_Stat, GA_ReferralStat log = logging.getLogger('ckanext.ga-report') @@ -70,19 +70,16 @@ entries = q.order_by('ga_stat.key').all() def clean_key(key, val): - if key in ['Average time on site', 'Pages per visit', 'Percent new visits']: + if key in ['Average time on site', 'Pages per visit', 'New visits']: val = "%.2f" % round(float(val), 2) if key == 'Average time on site': mins, secs = divmod(float(val), 60) hours, mins = divmod(mins, 60) val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) - if key == 'Percent new visits': - key = 'New visits' + if key == 'New visits': val = "%s%%" % val - if key in ['Bounces', 'Total pageviews']: + if key in ['Bounces', 'Total page views', 'Total visits']: val = int(val) - if key == 'Total pageviews': - key = 'Total page views' return key, val @@ -96,7 +93,7 @@ for e in entries: d[e.key].append(float(e.value)) for k, v in d.iteritems(): - if k in ['Bounces', 'Total pageviews']: + if k in ['Bounces', 'Total page views', 'Total visits']: v = sum(v) else: v = float(sum(v))/len(v) @@ -113,6 +110,30 @@ 'Languages': 'languages', 'Country': 'country' } + + def shorten_name(name, length=60): + return (name[:length] + '..') if len(name) > 60 else name + + def fill_out_url(url): + import urlparse + return urlparse.urljoin(g.site_url, url) + + c.social_referrer_totals, c.social_referrers = [], [] + q = model.Session.query(GA_ReferralStat) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('ga_referrer.count::int desc') + for entry in q.all(): + c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), + entry.source,entry.count)) + + q = model.Session.query(GA_ReferralStat.url, + func.sum(GA_ReferralStat.count).label('count')) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('count desc').group_by(GA_ReferralStat.url) + for entry in q.all(): + c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', + entry[1])) + browser_version_re = re.compile("(.*)\((.*)\)") for k, v in keys.iteritems(): @@ -157,7 +178,10 @@ # Get the total for each set of values and then set the value as # a percentage of the total - total = sum([num for _,num in entries]) + if k == 'Social sources': + total = sum([x for n,x in c.global_totals if n == 'Total visits']) + else: + total = sum([num for _,num in entries]) setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) return render('ga_report/site/index.html') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -1,7 +1,7 @@ import os import logging import datetime - +import collections from pylons import config import ga_model @@ -94,7 +94,6 @@ self.get_full_period_name(period_name, period_complete_day), start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) @@ -107,6 +106,33 @@ ga_model.update_publisher_stats(period_name) # about 30 seconds. self.sitewide_stats( period_name ) + + self.update_social_info(period_name, start_date, end_date) + + def update_social_info(self, period_name, start_date, end_date): + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + query = 'ga:hasSocialSourceReferral=~Yes$' + metrics = 'ga:entrances' + sort = '-ga:entrances' + + # Supported query params at + # https://developers.google.com/analytics/devguides/reporting/core/v3/reference + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters=query, + start_date=start_date, + metrics=metrics, + sort=sort, + dimensions="ga:landingPagePath,ga:socialNetwork", + max_results=10000, + end_date=end_date).execute() + data = collections.defaultdict(list) + rows = results.get('rows',[]) + for row in rows: + from ga_model import _normalize_url + data[_normalize_url(row[0])].append( (row[1], int(row[2]),) ) + ga_model.update_social(period_name, data) def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'): @@ -176,12 +202,12 @@ max_results=10000, end_date=end_date).execute() result_data = results.get('rows') - ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]}) - - results = self.service.data().ga().get( - ids='ga:' + self.profile_id, - start_date=start_date, - metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits', + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors', max_results=10000, end_date=end_date).execute() result_data = results.get('rows') @@ -189,7 +215,8 @@ 'Pages per visit': result_data[0][0], 'Bounces': result_data[0][1], 'Average time on site': result_data[0][2], - 'Percent new visits': result_data[0][3], + 'New visits': result_data[0][3], + 'Total visits': result_data[0][4], } ga_model.update_sitewide_stats(period_name, "Totals", data) --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -13,6 +13,8 @@ def make_uuid(): return unicode(uuid.uuid4()) +metadata = MetaData() + class GA_Url(object): @@ -21,20 +23,6 @@ for k,v in kwargs.items(): setattr(self, k, v) -class GA_Stat(object): - - def __init__(self, **kwargs): - for k,v in kwargs.items(): - setattr(self, k, v) - -class GA_Publisher(object): - - def __init__(self, **kwargs): - for k,v in kwargs.items(): - setattr(self, k, v) - - -metadata = MetaData() url_table = Table('ga_url', metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), @@ -47,6 +35,13 @@ ) mapper(GA_Url, url_table) + +class GA_Stat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + stat_table = Table('ga_stat', metadata, Column('id', types.UnicodeText, primary_key=True, default=make_uuid), @@ -56,6 +51,12 @@ Column('value', types.UnicodeText), ) mapper(GA_Stat, stat_table) + +class GA_Publisher(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) pub_table = Table('ga_publisher', metadata, Column('id', types.UnicodeText, primary_key=True, @@ -71,6 +72,24 @@ mapper(GA_Publisher, pub_table) +class GA_ReferralStat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +referrer_table = Table('ga_referrer', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('source', types.UnicodeText), + Column('url', types.UnicodeText), + Column('count', types.Integer), + ) +mapper(GA_ReferralStat, referrer_table) + + + def init_tables(): metadata.create_all(model.meta.engine) @@ -93,8 +112,9 @@ >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices') '/dataset/weekly_fuel_prices' ''' - url = re.sub('https?://(www\.)?data.gov.uk', '', url) - return url + # Deliberately leaving a / + url = url.replace('http:/','') + return '/' + '/'.join(url.split('/')[2:]) def _get_department_id_of_url(url): @@ -167,6 +187,33 @@ model.Session.commit() +def update_social(period_name, data): + # Clean up first. + model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).delete() + + for url,data in data.iteritems(): + for entry in data: + source = entry[0] + count = entry[1] + + item = model.Session.query(GA_ReferralStat).\ + filter(GA_ReferralStat.period_name==period_name).\ + filter(GA_ReferralStat.source==source).\ + filter(GA_ReferralStat.url==url).first() + if item: + item.count = item.count + count + model.Session.add(item) + else: + # create the row + values = {'id': make_uuid(), + 'period_name': period_name, + 'source': source, + 'url': url, + 'count': count, + } + model.Session.add(GA_ReferralStat(**values)) + model.Session.commit() def update_publisher_stats(period_name): """ --- a/ckanext/ga_report/templates/ga_report/ga_util.html +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -15,10 +15,26 @@ - +
- + + + + + + + + + + +
Name% ViewsSourceVisits
${name}${source}${count}
+ + + + + + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -25,7 +25,6 @@

Site Usage

- ${usage_nav('Site-wide', None)}
@@ -60,7 +59,16 @@
  • Versions
    • -
  • Social Networks
    • +
  • + Social Networks + + +
    • + +
  • Languages
  • Country
    • @@ -91,8 +99,14 @@
    ${stat_table(c.os_versions)}
    +
    +

    Number of visits to urls referred from social networks

    + ${social_table(c.social_referrer_totals)} +
    - ${stat_table(c.social_networks)} +

    Percentage of visits referred from these social networks

    + + ${stat_table(c.social_networks, 'Visits')}
    ${stat_table(c.languages)} @@ -118,6 +132,7 @@ e.preventDefault(); $(this).tab('show'); }) + alert(window.location.hash);
    Name% ${title}