projects / ckanext-ga-report.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Adding site analytics at /data/analytics
Adding site analytics at /data/analytics

- Allows for download as CSV
- Also refactored the publisher urls to allow index page and a read for a specific publisher

file:a/ckanext/ga_report/controller.py -> file:b/ckanext/ga_report/controller.py
import logging import logging
from ckan.lib.base import BaseController, c, render import operator
import report_model from ckan.lib.base import BaseController, c, render, request, response
   
  import sqlalchemy
  from sqlalchemy import func, cast, Integer
  import ckan.model as model
  from ga_model import GA_Url, GA_Stat
   
log = logging.getLogger('ckanext.ga-report') log = logging.getLogger('ckanext.ga-report')
   
   
  def _get_month_name(str):
  import calendar
  from time import strptime
  d = strptime('2012-10', '%Y-%m')
  return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)
   
   
  def _month_details(cls):
  months = []
  vals = model.Session.query(cls.period_name).distinct().all()
  for m in vals:
  months.append( (m[0], _get_month_name(m)))
  return sorted(months, key=operator.itemgetter(0), reverse=True)
   
   
class GaReport(BaseController): class GaReport(BaseController):
   
  def csv(self, month):
  import csv
   
  entries = model.Session.query(GA_Stat).\
  filter(GA_Stat.period_name==month).\
  order_by('GA_Stat.stat_name, GA_Stat.key').all()
   
  response.headers['Content-disposition'] = 'attachment; filename=dgu_analytics_%s.csv' % (month)
  response.headers['Content-Type'] = "text/csv; charset=utf-8"
   
  writer = csv.writer(response)
  writer.writerow(["Period", "Statistic", "Key", "Value"])
   
  for entry in entries:
  writer.writerow([entry.period_name.encode('utf-8'),
  entry.stat_name.encode('utf-8'),
  entry.key.encode('utf-8'),
  entry.value.encode('utf-8')])
   
def index(self): def index(self):
return render('index.html')  
   
  # Get the month details by fetching distinct values and determining the
  # month names from the values.
  c.months = _month_details(GA_Stat)
   
  # Work out which month to show, based on query params of the first item
  c.month = request.params.get('month', c.months[0][0] if c.months else '')
  c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
   
  entries = model.Session.query(GA_Stat).\
  filter(GA_Stat.stat_name=='Totals').\
  filter(GA_Stat.period_name==c.month).all()
  c.global_totals = [(s.key, s.value) for s in entries ]
   
  keys = {
  'Browser versions': 'browsers',
  'Operating Systems versions': 'os',
  'Social sources': 'social_networks',
  'Languages': 'languages',
  'Country': 'country'
  }
   
  for k, v in keys.iteritems():
  entries = model.Session.query(GA_Stat).\
  filter(GA_Stat.stat_name==k).\
  filter(GA_Stat.period_name==c.month).\
  order_by('ga_stat.value::int desc').all()
  setattr(c, v, [(s.key, s.value) for s in entries ])
   
   
  return render('ga_report/site/index.html')
   
   
  class GaPublisherReport(BaseController):
  """
  Displays the pageview and visit count for specific publishers based on
  the datasets associated with the publisher.
  """
   
  def index(self):
  # Get the month details by fetching distinct values and determining the
  # month names from the values.
  c.months = _month_details(GA_Url)
   
  # Work out which month to show, based on query params of the first item
  c.month = request.params.get('month', c.months[0][0] if c.months else '')
  c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
   
  connection = model.Session.connection()
  q = """
  select department_id, sum(pageviews::int) views, sum(visitors::int) visits
  from ga_url
  where department_id <> ''
  and not url like '/publisher/%%'
  and period_name=%s
  group by department_id order by views desc limit 20;
  """
  c.top_publishers = []
  res = connection.execute(q, c.month)
  for row in res:
  c.top_publishers.append((model.Group.get(row[0]), row[1], row[2]))
   
  return render('ga_report/publisher/index.html')
   
   
  def read(self, id):
  c.publisher = model.Group.get(id)
  c.top_packages = [] # package, dataset_views in c.top_packages
   
  # Get the month details by fetching distinct values and determining the
  # month names from the values.
  c.months = _month_details(GA_Url)
   
  # Work out which month to show, based on query params of the first item
  c.month = request.params.get('month', c.months[0][0] if c.months else '')
  c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
   
  entry = model.Session.query(GA_Url).\
  filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\
  filter(GA_Url.period_name==c.month).first()
  c.publisher_page_views = entry.pageviews if entry else 0
   
  entries = model.Session.query(GA_Url).\
  filter(GA_Url.department_id==c.publisher.name).\
  filter(GA_Url.period_name==c.month).\
  order_by('ga_url.pageviews::int desc')[:20]
  for entry in entries:
  if entry.url.startswith('/dataset/'):
  p = model.Package.get(entry.url[len('/dataset/'):])
  c.top_packages.append((p,entry.pageviews,entry.visitors))
   
  return render('ga_report/publisher/read.html')
   
file:a/ckanext/ga_report/download_analytics.py -> file:b/ckanext/ga_report/download_analytics.py
  import os
import logging import logging
import datetime import datetime
   
from pylons import config from pylons import config
   
import ga_model import ga_model
   
#from ga_client import GA #from ga_client import GA
   
log = logging.getLogger('ckanext.ga-report') log = logging.getLogger('ckanext.ga-report')
   
FORMAT_MONTH = '%Y-%m' FORMAT_MONTH = '%Y-%m'
   
class DownloadAnalytics(object): class DownloadAnalytics(object):
'''Downloads and stores analytics info''' '''Downloads and stores analytics info'''
   
def __init__(self, service=None, profile_id=None): def __init__(self, service=None, profile_id=None):
self.period = config['ga-report.period'] self.period = config['ga-report.period']
self.service = service self.service = service
self.profile_id = profile_id self.profile_id = profile_id
   
   
def all_(self): def all_(self):
self.since_date(datetime.datetime(2010, 1, 1)) self.since_date(datetime.datetime(2010, 1, 1))
   
def latest(self): def latest(self):
if self.period == 'monthly': if self.period == 'monthly':
# from first of this month to today # from first of this month to today
now = datetime.datetime.now() now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1) first_of_this_month = datetime.datetime(now.year, now.month, 1)
periods = ((now.strftime(FORMAT_MONTH), periods = ((now.strftime(FORMAT_MONTH),
now.day, now.day,
first_of_this_month, now),) first_of_this_month, now),)
else: else:
raise NotImplementedError raise NotImplementedError
self.download_and_store(periods) self.download_and_store(periods)
   
   
def since_date(self, since_date): def since_date(self, since_date):
assert isinstance(since_date, datetime.datetime) assert isinstance(since_date, datetime.datetime)
periods = [] # (period_name, period_complete_day, start_date, end_date) periods = [] # (period_name, period_complete_day, start_date, end_date)
if self.period == 'monthly': if self.period == 'monthly':
first_of_the_months_until_now = [] first_of_the_months_until_now = []
year = since_date.year year = since_date.year
month = since_date.month month = since_date.month
now = datetime.datetime.now() now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1) first_of_this_month = datetime.datetime(now.year, now.month, 1)
while True: while True:
first_of_the_month = datetime.datetime(year, month, 1) first_of_the_month = datetime.datetime(year, month, 1)
if first_of_the_month == first_of_this_month: if first_of_the_month == first_of_this_month:
periods.append((now.strftime(FORMAT_MONTH), periods.append((now.strftime(FORMAT_MONTH),
now.day, now.day,
first_of_this_month, now)) first_of_this_month, now))
break break
elif first_of_the_month < first_of_this_month: elif first_of_the_month < first_of_this_month:
in_the_next_month = first_of_the_month + datetime.timedelta(40) in_the_next_month = first_of_the_month + datetime.timedelta(40)
last_of_the_month = datetime.datetime(in_the_next_month.year, last_of_the_month = datetime.datetime(in_the_next_month.year,
in_the_next_month.month, 1)\ in_the_next_month.month, 1)\
- datetime.timedelta(1) - datetime.timedelta(1)
periods.append((now.strftime(FORMAT_MONTH), 0, periods.append((now.strftime(FORMAT_MONTH), 0,
first_of_the_month, last_of_the_month)) first_of_the_month, last_of_the_month))
else: else:
# first_of_the_month has got to the future somehow # first_of_the_month has got to the future somehow
break break
month += 1 month += 1
if month > 12: if month > 12:
year += 1 year += 1
month = 1 month = 1
else: else:
raise NotImplementedError raise NotImplementedError
self.download_and_store(periods) self.download_and_store(periods)
   
@staticmethod @staticmethod
def get_full_period_name(period_name, period_complete_day): def get_full_period_name(period_name, period_complete_day):
if period_complete_day: if period_complete_day:
return period_name + ' (up to %ith)' % period_complete_day return period_name + ' (up to %ith)' % period_complete_day
else: else:
return period_name return period_name
   
   
def download_and_store(self, periods): def download_and_store(self, periods):
for period_name, period_complete_day, start_date, end_date in periods: for period_name, period_complete_day, start_date, end_date in periods:
log.info('Downloading Analytics for period "%s" (%s - %s)', log.info('Downloading Analytics for period "%s" (%s - %s)',
self.get_full_period_name(period_name, period_complete_day), self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y %m %d'), start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d')) end_date.strftime('%Y %m %d'))
data = self.download(start_date, end_date) """
log.info('Storing Analytics for period "%s"', data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
  log.info('Storing Dataset Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day)) self.get_full_period_name(period_name, period_complete_day))
self.store(period_name, period_complete_day, data) self.store(period_name, period_complete_day, data, )
   
  data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
def download(self, start_date, end_date): log.info('Storing Publisher Analytics for period "%s"',
  self.get_full_period_name(period_name, period_complete_day))
  self.store(period_name, period_complete_day, data,)
  """
  ga_model.update_publisher_stats(period_name) # about 30 seconds.
  self.sitewide_stats( period_name )
   
   
  def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
'''Get data from GA for a given time period''' '''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d') start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:pagePath=~/dataset/[a-z0-9-]+$' query = 'ga:pagePath=%s$' % path
metrics = 'ga:uniquePageviews' metrics = 'ga:uniquePageviews, ga:visitors'
sort = '-ga:uniquePageviews' sort = '-ga:uniquePageviews'
   
# Supported query params at # Supported query params at
# https://developers.google.com/analytics/devguides/reporting/core/v3/reference # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
results = self.service.data().ga().get( results = self.service.data().ga().get(
ids='ga:' + self.profile_id, ids='ga:' + self.profile_id,
filters=query, filters=query,
start_date=start_date, start_date=start_date,
metrics=metrics, metrics=metrics,
sort=sort, sort=sort,
dimensions="ga:pagePath", dimensions="ga:pagePath",
max_results=10000, max_results=10000,
end_date=end_date).execute() end_date=end_date).execute()
   
  if os.getenv('DEBUG'):
import pprint import pprint
pprint.pprint(results) pprint.pprint(results)
print 'Total results: %s' % results.get('totalResults') print 'Total results: %s' % results.get('totalResults')
   
packages = [] packages = []
for entry in results.get('rows'): for entry in results.get('rows'):
(loc,size,) = entry (loc,pageviews,visits) = entry
packages.append( ('http:/' + loc,size, '',) ) # Temporary hack packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
return dict(url=packages) return dict(url=packages)
   
def store(self, period_name, period_complete_day, data): def store(self, period_name, period_complete_day, data):
if 'url' in data: if 'url' in data:
ga_model.update_url_stats(period_name, period_complete_day, data['url']) ga_model.update_url_stats(period_name, period_complete_day, data['url'])
   
  def sitewide_stats(self, period_name):
  import calendar
  year, month = period_name.split('-')
  _, last_day_of_month = calendar.monthrange(int(year), int(month))
   
  start_date = '%s-01' % period_name
  end_date = '%s-%s' % (period_name, last_day_of_month)
  print 'Sitewide_stats for %s (%s -> %s)' % (period_name, start_date, end_date)
   
  funcs = ['_totals_stats', '_social_stats', '_os_stats',
  '_locale_stats', '_browser_stats', '_mobile_stats']
  for f in funcs:
  print ' + Fetching %s stats' % f.split('_')[1]
  getattr(self, f)(start_date, end_date, period_name)
   
  def _get_results(result_data, f):
  data = {}
  for result in result_data:
  key = f(result)
  data[key] = data.get(key,0) + result[1]
  return data
   
  def _totals_stats(self, start_date, end_date, period_name):
  """ Fetches distinct totals, total pageviews etc """
  results = self.service.data().ga().get(
  ids='ga:' + self.profile_id,
  start_date=start_date,
  metrics='ga:uniquePageviews',
  sort='-ga:uniquePageviews',
  max_results=10000,
  end_date=end_date).execute()
  result_data = results.get('rows')
  ga_model.update_sitewide_stats(period_name, "Totals", {'Total pageviews': result_data[0][0]})
   
  results = self.service.data().ga().get(
  ids='ga:' + self.profile_id,
  start_date=start_date,
  metrics='ga:pageviewsPerVisit,ga:bounces,ga:avgTimeOnSite,ga:percentNewVisits',
  max_results=10000,
  end_date=end_date).execute()
  result_data = results.get('rows')
  data = {
  'Pages per visit': result_data[0][0],
  'Bounces': result_data[0][1],
  'Average time on site': result_data[0][2],
  'Percent new visits': result_data[0][3],
  }
  ga_model.update_sitewide_stats(period_name, "Totals", data)