From: Ross Jones Date: Tue, 30 Oct 2012 16:40:05 +0000 Subject: Moved minor nav into util file and implemented X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=d4ea62230eb7c98b2d8feef04739e843d42262b0 --- Moved minor nav into util file and implemented --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,5 +1,7 @@ +import re import logging import operator +import collections from ckan.lib.base import BaseController, c, render, request, response, abort import sqlalchemy @@ -30,9 +32,10 @@ def csv(self, month): import csv - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.period_name==month).\ - order_by('GA_Stat.stat_name, GA_Stat.key').all() + q = model.Session.query(GA_Stat) + if month != 'all': + q = q.filter(GA_Stat.period_name==month) + entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() response.headers['Content-Type'] = "text/csv; charset=utf-8" @@ -52,26 +55,46 @@ c.months = _month_details(GA_Stat) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name=='Totals').\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.key').all() - - c.global_totals = [] - for e in entries: - val = e.value - if e.key in ['Average time on site', 'Pages per visit', 'Percent new visits']: - val = "%.2f" % round(float(e.value), 2) - if e.key == 'Average time on site': + c.month_desc = 'all time' + c.month = request.params.get('month', '') + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals') + if c.month: + q = q.filter(GA_Stat.period_name==c.month) + entries = q.order_by('ga_stat.key').all() + + def clean_key(key, val): + if key in ['Average time on site', 'Pages per visit', 'Percent new visits']: + val = "%.2f" % round(float(val), 2) + if key == 'Average time on site': mins, secs = divmod(float(val), 60) hours, mins = divmod(mins, 60) val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) - e.key = '%s *' % e.key - c.global_totals.append((e.key, val)) - + key = '%s *' % key + if key in ['Bounces', 'Total pageviews']: + val = int(val) + return key, val + + c.global_totals = [] + if c.month: + for e in entries: + key, val = clean_key(e.key, e.value) + c.global_totals.append((key, val)) + else: + d = collections.defaultdict(list) + for e in entries: + d[e.key].append(float(e.value)) + for k, v in d.iteritems(): + if k in ['Bounces', 'Total pageviews']: + v = sum(v) + else: + v = float(sum(v))/len(v) + key, val = clean_key(k,v) + c.global_totals.append((key, val)) + c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) keys = { 'Browser versions': 'browsers', @@ -81,12 +104,45 @@ 'Country': 'country' } + browser_version_re = re.compile("(.*)\((.*)\)") for k, v in keys.iteritems(): - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name==k).\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.value::int desc').all() - setattr(c, v, [(s.key, s.value) for s in entries ]) + + def clean_field(key): + if k != 'Browser versions': + return key + m = browser_version_re.match(key) + browser = m.groups()[0].strip() + ver = m.groups()[1] + parts = ver.split('.') + if len(parts) > 1: + if parts[1][0] == '0': + ver = parts[0] + else: + ver = "%s.%s" % (parts[0],parts[1]) + if browser in ['Safari','Android Browser']: # Special case complex version nums + ver = parts[0] + if len(ver) > 2: + ver = "%s%sX" % (ver[0], ver[1]) + + return "%s (%s)" % (browser, ver,) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k) + if c.month: + entries = [] + q = q.filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc') + + d = collections.defaultdict(int) + for e in q.all(): + d[clean_field(e.key)] += int(e.value) + entries = [] + for key, val in d.iteritems(): + entries.append((key,val,)) + entries = sorted(entries, key=operator.itemgetter(1), reverse=True) + + setattr(c, v, [(k,v) for k,v in entries ]) + return render('ga_report/site/index.html') @@ -105,23 +161,31 @@ c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + c.month = request.params.get('month', '') + c.month_desc = 'all time' + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) connection = model.Session.connection() q = """ select department_id, sum(pageviews::int) views, sum(visitors::int) visits from ga_url - where department_id <> '' - and period_name=%s - group by department_id order by views desc limit 20; - """ + where department_id <> ''""" + if c.month: + q = q + """ + and period_name=%s + """ + q = q + """ + group by department_id order by views desc limit 20; + """ + # Add this back (before and period_name =%s) if you want to ignore publisher # homepage views # and not url like '/publisher/%%' c.top_publishers = [] res = connection.execute(q, c.month) + for row in res: c.top_publishers.append((model.Group.get(row[0]), row[1], row[2])) @@ -129,6 +193,7 @@ def read(self, id): + count = 20 c.publisher = model.Group.get(id) if not c.publisher: @@ -140,22 +205,50 @@ c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - entry = model.Session.query(GA_Url).\ - filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\ - filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 - - entries = model.Session.query(GA_Url).\ + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all time' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher.name) + if c.month: + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + else: + for e in q.all(): + c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + + + q = model.Session.query(GA_Url).\ filter(GA_Url.department_id==c.publisher.name).\ - filter(GA_Url.period_name==c.month).\ - order_by('ga_url.pageviews::int desc')[:20] - for entry in entries: - if entry.url.startswith('/dataset/'): + filter(GA_Url.url.like('/dataset/%')) + if c.month: + q = q.filter(GA_Url.period_name==c.month) + q = q.order_by('ga_url.pageviews::int desc') + + if c.month: + for entry in q[:count]: p = model.Package.get(entry.url[len('/dataset/'):]) c.top_packages.append((p,entry.pageviews,entry.visitors)) + else: + ds = {} + for entry in q.all(): + if len(ds) >= count: + break + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in ds: + ds[p] = {'views':0, 'visits': 0} + ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) + ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in ds.iteritems(): + results.append((k,v['views'],v['visits'])) + + c.top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) return render('ga_report/publisher/read.html') --- /dev/null +++ b/ckanext/ga_report/helpers.py @@ -1,1 +1,42 @@ +import logging +import operator +import ckan.lib.base as base +import ckan.model as model +_log = logging.getLogger(__name__) + +def most_popular_datasets(publisher, count=20): + from ckanext.ga_report.ga_model import GA_Url + + if not publisher: + _log.error("No valid publisher passed to 'most_popular_datasets'") + return "" + + datasets = {} + entries = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')).\ + order_by('ga_url.pageviews::int desc').all() + for entry in entries: + if len(datasets) < count: + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in datasets: + datasets[p] = {'views':0, 'visits': 0} + datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) + datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in datasets.iteritems(): + results.append((k,v['views'],v['visits'])) + + results = sorted(results, key=operator.itemgetter(1), reverse=True) + + ctx = { + 'dataset_count': len(datasets), + 'datasets': results, + + 'publisher': publisher + } + + return base.render_snippet('ga_report/publisher/popular.html', **ctx) + --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -2,19 +2,28 @@ import ckan.lib.helpers as h import ckan.plugins as p from ckan.plugins import implements, toolkit -#import gasnippet -#import commands -#import dbutil log = logging.getLogger('ckanext.ga-report') class GAReportPlugin(p.SingletonPlugin): implements(p.IConfigurer, inherit=True) implements(p.IRoutes, inherit=True) + implements(p.ITemplateHelpers, inherit=True) def update_config(self, config): toolkit.add_template_directory(config, 'templates') toolkit.add_public_directory(config, 'public') + + def get_helpers(self): + """ + A dictionary of extra helpers that will be available to provide + ga report info to templates. + """ + from ckanext.ga_report.helpers import most_popular_datasets + return { + 'ga_report_installed': lambda: True, + 'most_popular_datasets': most_popular_datasets, + } def after_map(self, map): map.connect( --- /dev/null +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -1,1 +1,37 @@ + + + + + + + +
${title}
+ + +
+
+ +
+
+ + + + --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -2,23 +2,41 @@ xmlns:i18n="http://genshi.edgewall.org/i18n" xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> + + Publisher Analytics for ${g.site_title}
  • Publishers

    -

    The table shows the top 20 publishers as recorded by page views of datasets owned by that publisher, and the number of visits to each publisher's home page.

    +

    + Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. +

    +

    + Note: this data does not include API calls. +

    +
  • +
  • +

    Download

    +

    + Download as CSV
    +

  • +
    -

    Publisher Analytics

    -

    The top 20 publishers

    +

    Site Usage

    + + ${usage_nav('Publishers', None)} +
    + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -3,22 +3,36 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> - Site analytics + + + Site usage
  • -

    Statistics

    -

    It is possible to export the analytics data as a CSV file, which contains all of the information for ${c.month_desc}

    +

    Site-wide

    +

    + Note: this data does not include API calls and some values have been rounded up to 2 decimal places. Where there are a large number of browser versions they have been grouped together. +

    +
  • +
  • +

    Download

    +

    + Download as CSV
    +

    +
  • -
    -

    Site statistics

    +

    Site Usage

    + + ${usage_nav('Site-wide', None)}