--- /dev/null
+++ b/ckanext/ga_report/controller.py
@@ -1,1 +1,142 @@
+import logging
+import operator
+from ckan.lib.base import BaseController, c, render, request, response
+import sqlalchemy
+from sqlalchemy import func, cast, Integer
+import ckan.model as model
+from ga_model import GA_Url, GA_Stat
+
+log = logging.getLogger('ckanext.ga-report')
+
+
+def _get_month_name(str):
+ import calendar
+ from time import strptime
+ d = strptime('2012-10', '%Y-%m')
+ return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)
+
+
+def _month_details(cls):
+ months = []
+ vals = model.Session.query(cls.period_name).distinct().all()
+ for m in vals:
+ months.append( (m[0], _get_month_name(m)))
+ return sorted(months, key=operator.itemgetter(0), reverse=True)
+
+
+class GaReport(BaseController):
+
+ def csv(self, month):
+ import csv
+
+ entries = model.Session.query(GA_Stat).\
+ filter(GA_Stat.period_name==month).\
+ order_by('GA_Stat.stat_name, GA_Stat.key').all()
+
+ response.headers['Content-disposition'] = 'attachment; filename=dgu_analytics_%s.csv' % (month)
+ response.headers['Content-Type'] = "text/csv; charset=utf-8"
+
+ writer = csv.writer(response)
+ writer.writerow(["Period", "Statistic", "Key", "Value"])
+
+ for entry in entries:
+ writer.writerow([entry.period_name.encode('utf-8'),
+ entry.stat_name.encode('utf-8'),
+ entry.key.encode('utf-8'),
+ entry.value.encode('utf-8')])
+
+ def index(self):
+
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Stat)
+
+ # Work out which month to show, based on query params of the first item
+ c.month = request.params.get('month', c.months[0][0] if c.months else '')
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ entries = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name=='Totals').\
+ filter(GA_Stat.period_name==c.month).all()
+ c.global_totals = [(s.key, s.value) for s in entries ]
+
+ keys = {
+ 'Browser versions': 'browsers',
+ 'Operating Systems versions': 'os',
+ 'Social sources': 'social_networks',
+ 'Languages': 'languages',
+ 'Country': 'country'
+ }
+
+ for k, v in keys.iteritems():
+ entries = model.Session.query(GA_Stat).\
+ filter(GA_Stat.stat_name==k).\
+ filter(GA_Stat.period_name==c.month).\
+ order_by('ga_stat.value::int desc').all()
+ setattr(c, v, [(s.key, s.value) for s in entries ])
+
+
+ return render('ga_report/site/index.html')
+
+
+class GaPublisherReport(BaseController):
+ """
+ Displays the pageview and visit count for specific publishers based on
+ the datasets associated with the publisher.
+ """
+
+ def index(self):
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Url)
+
+ # Work out which month to show, based on query params of the first item
+ c.month = request.params.get('month', c.months[0][0] if c.months else '')
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ connection = model.Session.connection()
+ q = """
+ select department_id, sum(pageviews::int) views, sum(visitors::int) visits
+ from ga_url
+ where department_id <> ''
+ and not url like '/publisher/%%'
+ and period_name=%s
+ group by department_id order by views desc limit 20;
+ """
+ c.top_publishers = []
+ res = connection.execute(q, c.month)
+ for row in res:
+ c.top_publishers.append((model.Group.get(row[0]), row[1], row[2]))
+
+ return render('ga_report/publisher/index.html')
+
+
+ def read(self, id):
+ c.publisher = model.Group.get(id)
+ c.top_packages = [] # package, dataset_views in c.top_packages
+
+ # Get the month details by fetching distinct values and determining the
+ # month names from the values.
+ c.months = _month_details(GA_Url)
+
+ # Work out which month to show, based on query params of the first item
+ c.month = request.params.get('month', c.months[0][0] if c.months else '')
+ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
+
+ entry = model.Session.query(GA_Url).\
+ filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\
+ filter(GA_Url.period_name==c.month).first()
+ c.publisher_page_views = entry.pageviews if entry else 0
+
+ entries = model.Session.query(GA_Url).\
+ filter(GA_Url.department_id==c.publisher.name).\
+ filter(GA_Url.period_name==c.month).\
+ order_by('ga_url.pageviews::int desc')[:20]
+ for entry in entries:
+ if entry.url.startswith('/dataset/'):
+ p = model.Package.get(entry.url[len('/dataset/'):])
+ c.top_packages.append((p,entry.pageviews,entry.visitors))
+
+ return render('ga_report/publisher/read.html')
+
--- /dev/null
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,1 +1,299 @@
-
+import os
+import logging
+import datetime
+
+from pylons import config
+
+import ga_model
+
+#from ga_client import GA
+
+log = logging.getLogger('ckanext.ga-report')
+
+FORMAT_MONTH = '%Y-%m'
+
+class DownloadAnalytics(object):
+ '''Downloads and stores analytics info'''
+
+ def __init__(self, service=None, profile_id=None):
+ self.period = config['ga-report.period']
+ self.service = service
+ self.profile_id = profile_id
+
+
+ def all_(self):
+ self.since_date(datetime.datetime(2010, 1, 1))
+
+ def latest(self):
+ if self.period == 'monthly':
+ # from first of this month to today
+ now = datetime.datetime.now()
+ first_of_this_month = datetime.datetime(now.year, now.month, 1)
+ periods = ((now.strftime(FORMAT_MONTH),
+ now.day,
+ first_of_this_month, now),)
+ else:
+ raise NotImplementedError
+ self.download_and_store(periods)
+
+
+ def since_date(self, since_date):
+ assert isinstance(since_date, datetime.datetime)
+ periods = [] # (period_name, period_complete_day, start_date, end_date)
+ if self.period == 'monthly':
+ first_of_the_months_until_now = []
+ year = since_date.year
+ month = since_date.month
+ now = datetime.datetime.now()
+ first_of_this_month = datetime.datetime(now.year, now.month, 1)
+ while True:
+ first_of_the_month = datetime.datetime(year, month, 1)
+ if first_of_the_month == first_of_this_month:
+ periods.append((now.strftime(FORMAT_MONTH),
+ now.day,
+ first_of_this_month, now))
+ break
+ elif first_of_the_month < first_of_this_month:
+ in_the_next_month = first_of_the_month + datetime.timedelta(40)
+ last_of_the_month = datetime.datetime(in_the_next_month.year,
+ in_the_next_month.month, 1)\
+ - datetime.timedelta(1)
+ periods.append((now.strftime(FORMAT_MONTH), 0,
+ first_of_the_month, last_of_the_month))
+ else:
+ # first_of_the_month has got to the future somehow
+ break
+ month += 1
+ if month > 12:
+ year += 1
+ month = 1
+ else:
+ raise NotImplementedError
+ self.download_and_store(periods)
+
+ @staticmethod
+ def get_full_period_name(period_name, period_complete_day):
+ if period_complete_day:
+ return period_name + ' (up to %ith)' % period_complete_day
+ else:
+ return period_name
+
+
+ def download_and_store(self, periods):
+ for period_name, period_complete_day, start_date, end_date in periods:
+ log.info('Downloading Analytics for period "%s" (%s - %s)',
+ self.get_full_period_name(period_name, period_complete_day),
+ start_date.strftime('%Y %m %d'),
+ end_date.strftime('%Y %m %d'))
+ """
+ data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
+ log.info('Storing Dataset Analytics for period "%s"',
+ self.get_full_period_name(period_name, period_complete_day))
+ self.store(period_name, period_complete_day, data, )
+
+ data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
+ log.info('Storing Publisher Analytics for period "%s"',
+ self.get_full_period_name(period_name, period_complete_day))
+ self.store(period_name, period_complete_day, data,)
+ """
+ ga_model.update_publisher_stats(period_name) # about 30 seconds.
+ self.sitewide_stats( period_name )
+
+
+ def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
+ '''Get data from GA for a given time period'''
+ start_date = start_date.strftime('%Y-%m-%d')
+ end_date = end_date.strftime('%Y-%m-%d')
+ query = 'ga:pagePath=%s$' % path
+ metrics = 'ga:uniquePageviews, ga:visitors'
+ sort = '-ga:uniquePageviews'
+
+ # Supported query params at
+ # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ filters=query,
+ start_date=start_date,
+ metrics=metrics,
+ sort=sort,
+ dimensions="ga:pagePath",
+ max_results=10000,
+ end_date=end_date).execute()
+
+ if os.getenv('DEBUG'):
+ import pprint
+ pprint.pprint(results)
+ print 'Total results: %s' % results.get('totalResults')
+
+ packages = []
+ for entry in results.get('rows'):
+ (loc,pageviews,visits) = entry
+ packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
+ return dict(url=packages)
+
+ def store(self, period_name, period_complete_day, data):
+ if 'url' in data:
+ ga_model.update_url_stats(period_name, period_complete_day, data['url'])
+
+ def sitewide_stats(self, period_name):
+ import calendar
+ year, month = period_name.split('-')
+ _, last_day_of_month = calendar.monthrange(int(year), int(month))
+
+ start_date = '%s-01' % period_name
+ end_date = '%s-%s' % (period_name, last_day_of_month)
+ print 'Sitewide_stats for %s (%s -> %s)' % (period_name, start_date, end_date)
+
+ funcs = ['_totals_stats', '_social_stats', '_os_stats',
+ '_locale_stats', '_browser_stats', '_mobile_stats']
+ for f in funcs:
+ print ' + Fetching %s stats' % f.split('_')[1]
+ getattr(self, f)(start_date, end_date, period_name)
+
+ def _get_results(result_data, f):
+ data = {}
+ for result in result_data:
+ key = f(result)
+ data[key] = data.get(key,0) + result[1]
+ return data
+
+ def _totals_stats(self, start_date, end_date, period_name):
+ """ Fetches distinct totals, total pageviews etc """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]})
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits',
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {
+ 'Pages per visit': result_data[0][0],
+ 'Bounces': result_data[0][1],
+ 'Average time on site': result_data[0][2],
+ 'Percent new visits': result_data[0][3],
+ }
+ ga_model.update_sitewide_stats(period_name, "Totals", data)
+
+
+ def _locale_stats(self, start_date, end_date, period_name):
+ """ Fetches stats about language and country """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:language,ga:country",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Languages", data)
+
+ data = {}
+ for result in result_data:
+ data[result[1]] = data.get(result[1], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Country", data)
+
+
+ def _social_stats(self, start_date, end_date, period_name):
+ """ Finds out which social sites people are referred from """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:socialNetwork,ga:referralPath",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ twitter_links = []
+ data = {}
+ for result in result_data:
+ if not result[0] == '(not set)':
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ if result[0] == 'Twitter':
+ twitter_links.append(result[1])
+ ga_model.update_sitewide_stats(period_name, "Social sources", data)
+
+
+ def _os_stats(self, start_date, end_date, period_name):
+ """ Operating system stats """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:operatingSystem,ga:operatingSystemVersion",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Operating Systems", data)
+
+ data = {}
+ for result in result_data:
+ key = "%s (%s)" % (result[0],result[1])
+ data[key] = result[2]
+ ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data)
+
+
+ def _browser_stats(self, start_date, end_date, period_name):
+ """ Information about browsers and browser versions """
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:browser,ga:browserVersion",
+ max_results=10000,
+ end_date=end_date).execute()
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Browsers", data)
+
+ data = {}
+ for result in result_data:
+ key = "%s (%s)" % (result[0], result[1])
+ data[key] = result[2]
+ ga_model.update_sitewide_stats(period_name, "Browser versions", data)
+
+
+ def _mobile_stats(self, start_date, end_date, period_name):
+ """ Info about mobile devices """
+
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ start_date=start_date,
+ metrics='ga:uniquePageviews',
+ sort='-ga:uniquePageviews',
+ dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",
+ max_results=10000,
+ end_date=end_date).execute()
+
+ result_data = results.get('rows')
+ data = {}
+ for result in result_data:
+ data[result[0]] = data.get(result[0], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Mobile brands", data)
+
+ data = {}
+ for result in result_data:
+ data[result[1]] = data.get(result[1], 0) + int(result[2])
+ ga_model.update_sitewide_stats(period_name, "Mobile devices", data)
+