--- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,5 +1,9 @@ +import re +import csv +import sys import logging import operator +import collections from ckan.lib.base import BaseController, c, render, request, response, abort import sqlalchemy @@ -30,11 +34,13 @@ def csv(self, month): import csv - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.period_name==month).\ - order_by('GA_Stat.stat_name, GA_Stat.key').all() + q = model.Session.query(GA_Stat) + if month != 'all': + q = q.filter(GA_Stat.period_name==month) + entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) writer = csv.writer(response) writer.writerow(["Period", "Statistic", "Key", "Value"]) @@ -52,42 +58,107 @@ c.months = _month_details(GA_Stat) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name=='Totals').\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.key').all() - - c.global_totals = [] - for e in entries: - val = e.value - if e.key in ['Average time on site', 'Pages per visit', 'Percent new visits']: - val = "%.2f" % round(float(e.value), 2) - if e.key == 'Average time on site': + c.month_desc = 'all time' + c.month = request.params.get('month', '') + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals') + if c.month: + q = q.filter(GA_Stat.period_name==c.month) + entries = q.order_by('ga_stat.key').all() + + def clean_key(key, val): + if key in ['Average time on site', 'Pages per visit', 'Percent new visits']: + val = "%.2f" % round(float(val), 2) + if key == 'Average time on site': mins, secs = divmod(float(val), 60) hours, mins = divmod(mins, 60) val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) - e.key = '%s *' % e.key - c.global_totals.append((e.key, val)) - + if key == 'Percent new visits': + key = 'New visits' + val = "%s%%" % val + if key in ['Bounces', 'Total pageviews']: + val = int(val) + if key == 'Total pageviews': + key = 'Total page views' + + return key, val + + c.global_totals = [] + if c.month: + for e in entries: + key, val = clean_key(e.key, e.value) + c.global_totals.append((key, val)) + else: + d = collections.defaultdict(list) + for e in entries: + d[e.key].append(float(e.value)) + for k, v in d.iteritems(): + if k in ['Bounces', 'Total pageviews']: + v = sum(v) + else: + v = float(sum(v))/len(v) + key, val = clean_key(k,v) + c.global_totals.append((key, val)) + c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) keys = { - 'Browser versions': 'browsers', - 'Operating Systems versions': 'os', + 'Browser versions': 'browser_versions', + 'Browsers': 'browsers', + 'Operating Systems versions': 'os_versions', + 'Operating Systems': 'os', 'Social sources': 'social_networks', 'Languages': 'languages', 'Country': 'country' } + browser_version_re = re.compile("(.*)\((.*)\)") for k, v in keys.iteritems(): - entries = model.Session.query(GA_Stat).\ - filter(GA_Stat.stat_name==k).\ - filter(GA_Stat.period_name==c.month).\ - order_by('ga_stat.value::int desc').all() - setattr(c, v, [(s.key, s.value) for s in entries ]) - + + def clean_field(key): + if k != 'Browser versions': + return key + m = browser_version_re.match(key) + browser = m.groups()[0].strip() + ver = m.groups()[1] + parts = ver.split('.') + if len(parts) > 1: + if parts[1][0] == '0': + ver = parts[0] + else: + ver = "%s.%s" % (parts[0],parts[1]) + if browser in ['Safari','Android Browser']: # Special case complex version nums + ver = parts[0] + if len(ver) > 2: + ver = "%s%sX" % (ver[0], ver[1]) + + return "%s (%s)" % (browser, ver,) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k) + if c.month: + entries = [] + q = q.filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc') + + d = collections.defaultdict(int) + for e in q.all(): + d[clean_field(e.key)] += int(e.value) + entries = [] + for key, val in d.iteritems(): + entries.append((key,val,)) + entries = sorted(entries, key=operator.itemgetter(1), reverse=True) + + def percent(num, total): + p = 100 * float(num)/float(total) + return "%.2f%%" % round(p, 2) + + # Get the total for each set of values and then set the value as + # a percentage of the total + total = sum([num for _,num in entries]) + setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) return render('ga_report/site/index.html') @@ -97,6 +168,45 @@ Displays the pageview and visit count for specific publishers based on the datasets associated with the publisher. """ + def csv(self, month): + + c.month = month if not month =='all' else '' + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for publisher,view,visit in self._get_publishers(None): + writer.writerow([publisher.title.encode('utf-8'), + view, + visit, + month]) + + + + def publisher_csv(self, id, month): + + c.month = month if not month =='all' else '' + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + + packages = self._get_packages(c.publisher) + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = \ + str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) + + for package,view,visit in packages: + writer.writerow([package.title.encode('utf-8'), + view, + visit, + month]) + + def index(self): @@ -110,6 +220,11 @@ if c.month: c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + c.top_publishers = self._get_publishers() + + return render('ga_report/publisher/index.html') + + def _get_publishers(self, limit=20): connection = model.Session.connection() q = """ select department_id, sum(pageviews::int) views, sum(visitors::int) visits @@ -120,54 +235,31 @@ and period_name=%s """ q = q + """ - group by department_id order by views desc limit 20; + group by department_id order by views desc """ + if limit: + q = q + " limit %s;" % (limit) # Add this back (before and period_name =%s) if you want to ignore publisher # homepage views # and not url like '/publisher/%%' - c.top_publishers = [] + top_publishers = [] res = connection.execute(q, c.month) for row in res: - c.top_publishers.append((model.Group.get(row[0]), row[1], row[2])) - - return render('ga_report/publisher/index.html') - - - def read(self, id): - count = 20 - - c.publisher = model.Group.get(id) - if not c.publisher: - abort(404, 'A publisher with that name could not be found') - c.top_packages = [] # package, dataset_views in c.top_packages - - # Get the month details by fetching distinct values and determining the - # month names from the values. - c.months = _month_details(GA_Url) - - # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', '') - if not c.month: - c.month_desc = 'all time' - else: - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - - c.publisher_page_views = 0 - q = model.Session.query(GA_Url).\ - filter(GA_Url.url=='/publisher/%s' % c.publisher.name) - if c.month: - entry = q.filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 - else: - for e in q.all(): - c.publisher_page_views = c.publisher_page_views + int(e.pageviews) - - + g = model.Group.get(row[0]) + if g: + top_publishers.append((g, row[1], row[2])) + return top_publishers + + def _get_packages(self, publisher, count=-1): + if count == -1: + count = sys.maxint + + top_packages = [] q = model.Session.query(GA_Url).\ - filter(GA_Url.department_id==c.publisher.name).\ + filter(GA_Url.department_id==publisher.name).\ filter(GA_Url.url.like('/dataset/%')) if c.month: q = q.filter(GA_Url.period_name==c.month) @@ -176,7 +268,7 @@ if c.month: for entry in q[:count]: p = model.Package.get(entry.url[len('/dataset/'):]) - c.top_packages.append((p,entry.pageviews,entry.visitors)) + top_packages.append((p,entry.pageviews,entry.visitors)) else: ds = {} for entry in q.all(): @@ -192,7 +284,40 @@ for k, v in ds.iteritems(): results.append((k,v['views'],v['visits'])) - c.top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + return top_packages + + + def read(self, id): + count = 20 + + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + c.top_packages = [] # package, dataset_views in c.top_packages + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all time' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher.name) + if c.month: + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + else: + for e in q.all(): + c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + + c.top_packages = self._get_packages(c.publisher, 20) return render('ga_report/publisher/read.html')