gitphp 0.2.9.1 :: ckanext-ga-report.git/commitdiff

Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-ga-report

7 files changed: (show all)
ckanext/ga_report/controller.py
ckanext/ga_report/download_analytics.py
ckanext/ga_report/helpers.py
ckanext/ga_report/plugin.py
ckanext/ga_report/templates/ga_report/publisher/read.html
ckanext/ga_report/templates/ga_report/site/index.html
ckanext/ga_report/templates/ga_report/site/layout_base.html

file:a/ckanext/ga_report/controller.py -> file:b/ckanext/ga_report/controller.py

import re	import re
import csv	import csv
import sys	import sys
import json	import json
import logging	import logging
import operator	import operator
import collections	import collections
from ckan.lib.base import (BaseController, c, g, render, request, response, abort)	from ckan.lib.base import (BaseController, c, g, render, request, response, abort)

import sqlalchemy	import sqlalchemy
from sqlalchemy import func, cast, Integer	from sqlalchemy import func, cast, Integer
import ckan.model as model	import ckan.model as model
from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher	from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher

log = logging.getLogger('ckanext.ga-report')	log = logging.getLogger('ckanext.ga-report')

DOWNLOADS_AVAILABLE_FROM = '2012-12'	DOWNLOADS_AVAILABLE_FROM = '2012-12'

def _get_month_name(strdate):	def _get_month_name(strdate):
import calendar	import calendar
from time import strptime	from time import strptime
d = strptime(strdate, '%Y-%m')	d = strptime(strdate, '%Y-%m')
return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)	return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year)

def _get_unix_epoch(strdate):	def _get_unix_epoch(strdate):
from time import strptime,mktime	from time import strptime,mktime
d = strptime(strdate, '%Y-%m')	d = strptime(strdate, '%Y-%m')
return int(mktime(d))	return int(mktime(d))

def _month_details(cls, stat_key=None):	def _month_details(cls, stat_key=None):
'''	'''
Returns a list of all the periods for which we have data, unfortunately	Returns a list of all the periods for which we have data, unfortunately
knows too much about the type of the cls being passed as GA_Url has a	knows too much about the type of the cls being passed as GA_Url has a
more complex query	more complex query

This may need extending if we add a period_name to the stats	This may need extending if we add a period_name to the stats
'''	'''
months = []	months = []
day = None	day = None

q = model.Session.query(cls.period_name,cls.period_complete_day)\	q = model.Session.query(cls.period_name,cls.period_complete_day)\
.filter(cls.period_name!='All').distinct(cls.period_name)	.filter(cls.period_name!='All').distinct(cls.period_name)
if stat_key:	if stat_key:
q= q.filter(cls.stat_name==stat_key)	q= q.filter(cls.stat_name==stat_key)

vals = q.order_by("period_name desc").all()	vals = q.order_by("period_name desc").all()

if vals and vals[0][1]:	if vals and vals[0][1]:
day = int(vals[0][1])	day = int(vals[0][1])
ordinal = 'th' if 11 <= day <= 13 \	ordinal = 'th' if 11 <= day <= 13 \
else {1:'st',2:'nd',3:'rd'}.get(day % 10, 'th')	else {1:'st',2:'nd',3:'rd'}.get(day % 10, 'th')
day = "{day}{ordinal}".format(day=day, ordinal=ordinal)	day = "{day}{ordinal}".format(day=day, ordinal=ordinal)

for m in vals:	for m in vals:
months.append( (m[0], _get_month_name(m[0])))	months.append( (m[0], _get_month_name(m[0])))

return months, day	return months, day


class GaReport(BaseController):	class GaReport(BaseController):

def csv(self, month):	def csv(self, month):
import csv	import csv

q = model.Session.query(GA_Stat).filter(GA_Stat.stat_name!='Downloads')	q = model.Session.query(GA_Stat).filter(GA_Stat.stat_name!='Downloads')
if month != 'all':	if month != 'all':
q = q.filter(GA_Stat.period_name==month)	q = q.filter(GA_Stat.period_name==month)
entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all()	entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all()

response.headers['Content-Type'] = "text/csv; charset=utf-8"	response.headers['Content-Type'] = "text/csv; charset=utf-8"
response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,))	response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,))

writer = csv.writer(response)	writer = csv.writer(response)
writer.writerow(["Period", "Statistic", "Key", "Value"])	writer.writerow(["Period", "Statistic", "Key", "Value"])

for entry in entries:	for entry in entries:
writer.writerow([entry.period_name.encode('utf-8'),	writer.writerow([entry.period_name.encode('utf-8'),
entry.stat_name.encode('utf-8'),	entry.stat_name.encode('utf-8'),
entry.key.encode('utf-8'),	entry.key.encode('utf-8'),
entry.value.encode('utf-8')])	entry.value.encode('utf-8')])


def index(self):	def index(self):

# Get the month details by fetching distinct values and determining the	# Get the month details by fetching distinct values and determining the
# month names from the values.	# month names from the values.
c.months, c.day = _month_details(GA_Stat)	c.months, c.day = _month_details(GA_Stat)

# Work out which month to show, based on query params of the first item	# Work out which month to show, based on query params of the first item
c.month_desc = 'all months'	c.month_desc = 'all months'
c.month = request.params.get('month', '')	c.month = request.params.get('month', '')
if c.month:	if c.month:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])	c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])

q = model.Session.query(GA_Stat).\	q = model.Session.query(GA_Stat).\
filter(GA_Stat.stat_name=='Totals')	filter(GA_Stat.stat_name=='Totals')
if c.month:	if c.month:
q = q.filter(GA_Stat.period_name==c.month)	q = q.filter(GA_Stat.period_name==c.month)
entries = q.order_by('ga_stat.key').all()	entries = q.order_by('ga_stat.key').all()

def clean_key(key, val):	def clean_key(key, val):
if key in ['Average time on site', 'Pages per visit', 'New visits', 'Bounce rate (home page)']:	if key in ['Average time on site', 'Pages per visit', 'New visits', 'Bounce rate (home page)']:
val = "%.2f" % round(float(val), 2)	val = "%.2f" % round(float(val), 2)
if key == 'Average time on site':	if key == 'Average time on site':
mins, secs = divmod(float(val), 60)	mins, secs = divmod(float(val), 60)
hours, mins = divmod(mins, 60)	hours, mins = divmod(mins, 60)
val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)	val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
if key in ['New visits','Bounce rate (home page)']:	if key in ['New visits','Bounce rate (home page)']:
val = "%s%%" % val	val = "%s%%" % val
if key in ['Total page views', 'Total visits']:	if key in ['Total page views', 'Total visits']:
val = int(val)	val = int(val)

return key, val	return key, val

# Query historic values for sparkline rendering	# Query historic values for sparkline rendering
sparkline_query = model.Session.query(GA_Stat)\	sparkline_query = model.Session.query(GA_Stat)\
.filter(GA_Stat.stat_name=='Totals')\	.filter(GA_Stat.stat_name=='Totals')\
.order_by(GA_Stat.period_name)	.order_by(GA_Stat.period_name)
sparkline_data = {}	sparkline_data = {}
for x in sparkline_query:	for x in sparkline_query:
sparkline_data[x.key] = sparkline_data.get(x.key,[])	sparkline_data[x.key] = sparkline_data.get(x.key,[])
key, val = clean_key(x.key,float(x.value))	key, val = clean_key(x.key,float(x.value))
tooltip = '%s: %s' % (_get_month_name(x.period_name), val)	tooltip = '%s: %s' % (_get_month_name(x.period_name), val)
sparkline_data[x.key].append( (tooltip,x.value) )	sparkline_data[x.key].append( (tooltip,x.value) )
# Trim the latest month, as it looks like a huge dropoff	# Trim the latest month, as it looks like a huge dropoff
for key in sparkline_data:	for key in sparkline_data:
sparkline_data[key] = sparkline_data[key][:-1]	sparkline_data[key] = sparkline_data[key][:-1]

c.global_totals = []	c.global_totals = []
if c.month:	if c.month:
for e in entries:	for e in entries:
key, val = clean_key(e.key, e.value)	key, val = clean_key(e.key, e.value)
sparkline = sparkline_data[e.key]	sparkline = sparkline_data[e.key]
c.global_totals.append((key, val, sparkline))	c.global_totals.append((key, val, sparkline))
else:	else:
d = collections.defaultdict(list)	d = collections.defaultdict(list)
for e in entries:	for e in entries:
d[e.key].append(float(e.value))	d[e.key].append(float(e.value))
for k, v in d.iteritems():	for k, v in d.iteritems():
if k in ['Total page views', 'Total visits']:	if k in ['Total page views', 'Total visits']:
v = sum(v)	v = sum(v)
else:	else:
v = float(sum(v))/float(len(v))	v = float(sum(v))/float(len(v))
sparkline = sparkline_data[k]	sparkline = sparkline_data[k]
key, val = clean_key(k,v)	key, val = clean_key(k,v)

c.global_totals.append((key, val, sparkline))	c.global_totals.append((key, val, sparkline))
# Sort the global totals into a more pleasant order	# Sort the global totals into a more pleasant order
def sort_func(x):	def sort_func(x):
key = x[0]	key = x[0]
total_order = ['Total page views','Total visits','Pages per visit']	total_order = ['Total page views','Total visits','Pages per visit']
if key in total_order:	if key in total_order:
return total_order.index(key)	return total_order.index(key)
return 999	return 999
c.global_totals = sorted(c.global_totals, key=sort_func)	c.global_totals = sorted(c.global_totals, key=sort_func)

keys = {	keys = {
'Browser versions': 'browser_versions',	'Browser versions': 'browser_versions',
'Browsers': 'browsers',	'Browsers': 'browsers',
'Operating Systems versions': 'os_versions',	'Operating Systems versions': 'os_versions',
'Operating Systems': 'os',	'Operating Systems': 'os',
'Social sources': 'social_networks',	'Social sources': 'social_networks',
'Languages': 'languages',	'Languages': 'languages',
'Country': 'country'	'Country': 'country'
}	}

def shorten_name(name, length=60):	def shorten_name(name, length=60):
return (name[:length] + '..') if len(name) > 60 else name	return (name[:length] + '..') if len(name) > 60 else name

def fill_out_url(url):	def fill_out_url(url):
import urlparse	import urlparse
return urlparse.urljoin(g.site_url, url)	return urlparse.urljoin(g.site_url, url)

c.social_referrer_totals, c.social_referrers = [], []	c.social_referrer_totals, c.social_referrers = [], []
q = model.Session.query(GA_ReferralStat)	q = model.Session.query(GA_ReferralStat)
q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q	q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
q = q.order_by('ga_referrer.count::int desc')	q = q.order_by('ga_referrer.count::int desc')
for entry in q.all():	for entry in q.all():
c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url),	c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url),
entry.source,entry.count))	entry.source,entry.count))

q = model.Session.query(GA_ReferralStat.url,	q = model.Session.query(GA_ReferralStat.url,
func.sum(GA_ReferralStat.count).label('count'))	func.sum(GA_ReferralStat.count).label('count'))
q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q	q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q
q = q.order_by('count desc').group_by(GA_ReferralStat.url)	q = q.order_by('count desc').group_by(GA_ReferralStat.url)
for entry in q.all():	for entry in q.all():
c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'',	c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'',
entry[1]))	entry[1]))

for k, v in keys.iteritems():	for k, v in keys.iteritems():
q = model.Session.query(GA_Stat).\	q = model.Session.query(GA_Stat).\
filter(GA_Stat.stat_name==k).\	filter(GA_Stat.stat_name==k).\
order_by(GA_Stat.period_name)	order_by(GA_Stat.period_name)
# Buffer the tabular data	# Buffer the tabular data
if c.month:	if c.month:
entries = []	entries = []
q = q.filter(GA_Stat.period_name==c.month).\	q = q.filter(GA_Stat.period_name==c.month).\
order_by('ga_stat.value::int desc')	order_by('ga_stat.value::int desc')
d = collections.defaultdict(int)	d = collections.defaultdict(int)
for e in q.all():	for e in q.all():
d[e.key] += int(e.value)	d[e.key] += int(e.value)
entries = []	entries = []
for key, val in d.iteritems():	for key, val in d.iteritems():
entries.append((key,val,))	entries.append((key,val,))
entries = sorted(entries, key=operator.itemgetter(1), reverse=True)	entries = sorted(entries, key=operator.itemgetter(1), reverse=True)

# Run a query on all months to gather graph data	# Run a query on all months to gather graph data
graph_query = model.Session.query(GA_Stat).\	graph_query = model.Session.query(GA_Stat).\
filter(GA_Stat.stat_name==k).\	filter(GA_Stat.stat_name==k).\
order_by(GA_Stat.period_name)	order_by(GA_Stat.period_name)
graph_dict = {}	graph_dict = {}
for stat in graph_query:	for stat in graph_query:
graph_dict[ stat.key ] = graph_dict.get(stat.key,{	graph_dict[ stat.key ] = graph_dict.get(stat.key,{
'name':stat.key,	'name':stat.key,
'raw': {}	'raw': {}
})	})
graph_dict[ stat.key ]['raw'][stat.period_name] = float(stat.value)	graph_dict[ stat.key ]['raw'][stat.period_name] = float(stat.value)
stats_in_table = [x[0] for x in entries]	stats_in_table = [x[0] for x in entries]
stats_not_in_table = set(graph_dict.keys()) - set(stats_in_table)	stats_not_in_table = set(graph_dict.keys()) - set(stats_in_table)
stats = stats_in_table + sorted(list(stats_not_in_table))	stats = stats_in_table + sorted(list(stats_not_in_table))
graph = [graph_dict[x] for x in stats]	graph = [graph_dict[x] for x in stats]
setattr(c, v+'_graph', json.dumps( _to_rickshaw(graph,percentageMode=True) ))	setattr(c, v+'_graph', json.dumps( _to_rickshaw(graph,percentageMode=True) ))

# Get the total for each set of values and then set the value as	# Get the total for each set of values and then set the value as
# a percentage of the total	# a percentage of the total
if k == 'Social sources':	if k == 'Social sources':
total = sum([x for n,x,graph in c.global_totals if n == 'Total visits'])	total = sum([x for n,x,graph in c.global_totals if n == 'Total visits'])
else:	else:
total = sum([num for _,num in entries])	total = sum([num for _,num in entries])
setattr(c, v, [(k,_percent(v,total)) for k,v in entries ])	setattr(c, v, [(k,_percent(v,total)) for k,v in entries ])

return render('ga_report/site/index.html')	return render('ga_report/site/index.html')


class GaDatasetReport(BaseController):	class GaDatasetReport(BaseController):
"""	"""
Displays the pageview and visit count for datasets	Displays the pageview and visit count for datasets
with options to filter by publisher and time period.	with options to filter by publisher and time period.
"""	"""
def publisher_csv(self, month):	def publisher_csv(self, month):
'''	'''
Returns a CSV of each publisher with the total number of dataset	Returns a CSV of each publisher with the total number of dataset
views & visits.	views & visits.
'''	'''
c.month = month if not month == 'all' else ''	c.month = month if not month == 'all' else ''
response.headers['Content-Type'] = "text/csv; charset=utf-8"	response.headers['Content-Type'] = "text/csv; charset=utf-8"
response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,))	response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,))

writer = csv.writer(response)	writer = csv.writer(response)
writer.writerow(["Publisher Title", "Publisher Name", "Views", "Visits", "Period Name"])	writer.writerow(["Publisher Title", "Publisher Name", "Views", "Visits", "Period Name"])

top_publishers = _get_top_publishers(limit=None)	top_publishers = _get_top_publishers(limit=None)

for publisher,view,visit in top_publishers:	for publisher,view,visit in top_publishers:
writer.writerow([publisher.title.encode('utf-8'),	writer.writerow([publisher.title.encode('utf-8'),
publisher.name.encode('utf-8'),	publisher.name.encode('utf-8'),
view,	view,
visit,	visit,
month])	month])

def dataset_csv(self, id='all', month='all'):	def dataset_csv(self, id='all', month='all'):
'''	'''
Returns a CSV with the number of views & visits for each dataset.	Returns a CSV with the number of views & visits for each dataset.

:param id: A Publisher ID or None if you want for all	:param id: A Publisher ID or None if you want for all
:param month: The time period, or 'all'	:param month: The time period, or 'all'
'''	'''
c.month = month if not month == 'all' else ''	c.month = month if not month == 'all' else ''
if id != 'all':	if id != 'all':
c.publisher = model.Group.get(id)	c.publisher = model.Group.get(id)
if not c.publisher:	if not c.publisher:
abort(404, 'A publisher with that name could not be found')	abort(404, 'A publisher with that name could not be found')

packages = self._get_packages(publisher=c.publisher, month=c.month)	packages = self._get_packages(publisher=c.publisher, month=c.month)
response.headers['Content-Type'] = "text/csv; charset=utf-8"	response.headers['Content-Type'] = "text/csv; charset=utf-8"
response.headers['Content-Disposition'] = \	response.headers['Content-Disposition'] = \
str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,))	str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,))

writer = csv.writer(response)	writer = csv.writer(response)
writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"])	writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"])

for package,view,visit,downloads in packages:	for package,view,visit,downloads in packages:
writer.writerow([package.title.encode('utf-8'),	writer.writerow([package.title.encode('utf-8'),
package.name.encode('utf-8'),	package.name.encode('utf-8'),
view,	view,
visit,	visit,
downloads,	downloads,
month])	month])

def publishers(self):	def publishers(self):
'''A list of publishers and the number of views/visits for each'''	'''A list of publishers and the number of views/visits for each'''

# Get the month details by fetching distinct values and determining the	# Get the month details by fetching distinct values and determining the
# month names from the values.	# month names from the values.
c.months, c.day = _month_details(GA_Url)	c.months, c.day = _month_details(GA_Url)

# Work out which month to show, based on query params of the first item	# Work out which month to show, based on query params of the first item
c.month = request.params.get('month', '')	c.month = request.params.get('month', '')
c.month_desc = 'all months'	c.month_desc = 'all months'
if c.month:	if c.month:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])	c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])

c.top_publishers = _get_top_publishers()	c.top_publishers = _get_top_publishers()
graph_data = _get_top_publishers_graph()	graph_data = _get_top_publishers_graph()
c.top_publishers_graph = json.dumps( _to_rickshaw(graph_data) )	c.top_publishers_graph = json.dumps( _to_rickshaw(graph_data) )

x = render('ga_report/publisher/index.html')	x = render('ga_report/publisher/index.html')

return x	return x

def _get_packages(self, publisher=None, month='', count=-1):	def _get_packages(self, publisher=None, month='', count=-1):
'''Returns the datasets in order of views'''	'''Returns the datasets in order of views'''
have_download_data = True	have_download_data = True
month = month or 'All'	month = month or 'All'
if month != 'All':	if month != 'All':
have_download_data = month >= DOWNLOADS_AVAILABLE_FROM	have_download_data = month >= DOWNLOADS_AVAILABLE_FROM

q = model.Session.query(GA_Url,model.Package)\	q = model.Session.query(GA_Url,model.Package)\
.filter(model.Package.name==GA_Url.package_id)\	.filter(model.Package.name==GA_Url.package_id)\
.filter(GA_Url.url.like('/dataset/%'))	.filter(GA_Url.url.like('/dataset/%'))
if publisher:	if publisher:
q = q.filter(GA_Url.department_id==publisher.name)	q = q.filter(GA_Url.department_id==publisher.name)
q = q.filter(GA_Url.period_name==month)	q = q.filter(GA_Url.period_name==month)
q = q.order_by('ga_url.pageviews::int desc')	q = q.order_by('ga_url.pageviews::int desc')
top_packages = []	top_packages = []
if count == -1:	if count == -1:
entries = q.all()	entries = q.all()
else:	else:
entries = q.limit(count)	entries = q.limit(count)

for entry,package in entries:	for entry,package in entries:
if package:	if package:
# Downloads ....	# Downloads ....
if have_download_data:	if have_download_data:
dls = model.Session.query(GA_Stat).\	dls = model.Session.query(GA_Stat).\
filter(GA_Stat.stat_name=='Downloads').\	filter(GA_Stat.stat_name=='Downloads').\
filter(GA_Stat.key==package.name)	filter(GA_Stat.key==package.name)
if month != 'All': # Fetch everything unless the month is specific	if month != 'All': # Fetch everything unless the month is specific
dls = dls.filter(GA_Stat.period_name==month)	dls = dls.filter(GA_Stat.period_name==month)
downloads = 0	downloads = 0
for x in dls:	for x in dls:
downloads += int(x.value)	downloads += int(x.value)
else:	else:
downloads = 'No data'	downloads = 'No data'
top_packages.append((package, entry.pageviews, entry.visits, downloads))	if package.private == False:
	top_packages.append((package, entry.pageviews, entry.visits, downloads))
else:	else:
log.warning('Could not find package associated package')	log.warning('Could not find package associated package')

return top_packages	return top_packages

def read(self):	def read(self):
'''	'''
Lists the most popular datasets across all publishers	Lists the most popular datasets across all publishers
'''	'''
return self.read_publisher(None)	return self.read_publisher(None)

def read_publisher(self, id):	def read_publisher(self, id):
'''	'''
Lists the most popular datasets for a publisher (or across all publishers)	Lists the most popular datasets for a publisher (or across all publishers)
'''	'''
count = 20	count = 20

c.publishers = _get_publishers()	c.publishers = _get_publishers()

id = request.params.get('publisher', id)	id = request.params.get('publisher', id)
if id and id != 'all':	if id and id != 'all':
c.publisher = model.Group.get(id)	c.publisher = model.Group.get(id)
if not c.publisher:	if not c.publisher:
abort(404, 'A publisher with that name could not be found')	abort(404, 'A publisher with that name could not be found')
c.publisher_name = c.publisher.name	c.publisher_name = c.publisher.name
c.top_packages = [] # package, dataset_views in c.top_packages	c.top_packages = [] # package, dataset_views in c.top_packages

# Get the month details by fetching distinct values and determining the	# Get the month details by fetching distinct values and determining the
# month names from the values.	# month names from the values.
c.months, c.day = _month_details(GA_Url)	c.months, c.day = _month_details(GA_Url)

# Work out which month to show, based on query params of the first item	# Work out which month to show, based on query params of the first item
c.month = request.params.get('month', '')	c.month = request.params.get('month', '')
if not c.month:	if not c.month:
c.month_desc = 'all months'	c.month_desc = 'all months'
else:	else:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])	c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])

month = c.month or 'All'	month = c.month or 'All'
c.publisher_page_views = 0	c.publisher_page_views = 0
q = model.Session.query(GA_Url).\	q = model.Session.query(GA_Url).\
filter(GA_Url.url=='/publisher/%s' % c.publisher_name)	filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
entry = q.filter(GA_Url.period_name==c.month).first()	entry = q.filter(GA_Url.period_name==c.month).first()
c.publisher_page_views = entry.pageviews if entry else 0	c.publisher_page_views = entry.pageviews if entry else 0

c.top_packages = self._get_packages(publisher=c.publisher, count=20, month=c.month)	c.top_packages = self._get_packages(publisher=c.publisher, count=20, month=c.month)

# Graph query	# Graph query
top_packages_all_time = self._get_packages(publisher=c.publisher, count=20, month='All')	top_packages_all_time = self._get_packages(publisher=c.publisher, count=20, month='All')
top_package_names = [ x[0].name for x in top_packages_all_time ]	top_package_names = [ x[0].name for x in top_packages_all_time ]
graph_query = model.Session.query(GA_Url,model.Package)\	graph_query = model.Session.query(GA_Url,model.Package)\
.filter(model.Package.name==GA_Url.package_id)\	.filter(model.Package.name==GA_Url.package_id)\
.filter(GA_Url.url.like('/dataset/%'))\	.filter(GA_Url.url.like('/dataset/%'))\
.filter(GA_Url.package_id.in_(top_package_names))	.filter(GA_Url.package_id.in_(top_package_names))
all_series = {}	all_series = {}
for entry,package in graph_query:	for entry,package in graph_query:
if not package: continue	if not package: continue
if entry.period_name=='All': continue	if entry.period_name=='All': continue
all_series[package.name] = all_series.get(package.name,{	all_series[package.name] = all_series.get(package.name,{
'name':package.title,	'name':package.title,
'raw': {}	'raw': {}
})	})
all_series[package.name]['raw'][entry.period_name] = int(entry.pageviews)	all_series[package.name]['raw'][entry.period_name] = int(entry.pageviews)
graph = [ all_series[series_name] for series_name in top_package_names ]	graph = [ all_series[series_name] for series_name in top_package_names ]
c.graph_data = json.dumps( _to_rickshaw(graph) )	c.graph_data = json.dumps( _to_rickshaw(graph) )

return render('ga_report/publisher/read.html')	return render('ga_report/publisher/read.html')

def _to_rickshaw(data, percentageMode=False):	def _to_rickshaw(data, percentageMode=False):
if data==[]:	if data==[]:
return data	return data
# x-axis is every month in c.months. Note that data might not exist	# x-axis is every month in c.months. Note that data might not exist
# for entire history, eg. for recently-added datasets	# for entire history, eg. for recently-added datasets
x_axis = [x[0] for x in c.months]	x_axis = [x[0] for x in c.months]
x_axis.reverse() # Ascending order	x_axis.reverse() # Ascending order
x_axis = x_axis[:-1] # Remove latest month	x_axis = x_axis[:-1] # Remove latest month
totals = {}	totals = {}
for series in data:	for series in data:
series['data'] = []	series['data'] = []
for x_string in x_axis:	for x_string in x_axis:
x = _get_unix_epoch( x_string )	x = _get_unix_epoch( x_string )
y = series['raw'].get(x_string,0)	y = series['raw'].get(x_string,0)
series['data'].append({'x':x,'y':y})	series['data'].append({'x':x,'y':y})
totals[x] = totals.get(x,0)+y	totals[x] = totals.get(x,0)+y
if not percentageMode:	if not percentageMode:
return data	return data
# Turn all data into percentages	# Turn all data into percentages
# Roll insignificant series into a catch-all	# Roll insignificant series into a catch-all
THRESHOLD = 1	THRESHOLD = 1
raw_data = data	raw_data = data
data = []	data = []
for series in raw_data:	for series in raw_data:
for point in series['data']:	for point in series['data']:
percentage = (100*float(point['y'])) / totals[point['x']]	percentage = (100*float(point['y'])) / totals[point['x']]
if not (series in data) and percentage>THRESHOLD:	if not (series in data) and percentage>THRESHOLD:
data.append(series)	data.append(series)
point['y'] = percentage	point['y'] = percentage
others = [ x for x in raw_data if not (x in data) ]	others = [ x for x in raw_data if not (x in data) ]
if len(others):	if len(others):
data_other = []	data_other = []
for i in range(len(x_axis)):	for i in range(len(x_axis)):
x = _get_unix_epoch(x_axis[i])	x = _get_unix_epoch(x_axis[i])
y = 0	y = 0
for series in others:	for series in others:
y += series['data'][i]['y']	y += series['data'][i]['y']
data_other.append({'x':x,'y':y})	data_other.append({'x':x,'y':y})
data.append({	data.append({
'name':'Other',	'name':'Other',
'data': data_other	'data': data_other
})	})
return data	return data


def _get_top_publishers(limit=20):	def _get_top_publishers(limit=20):
'''	'''
Returns a list of the top 20 publishers by dataset visits.	Returns a list of the top 20 publishers by dataset visits.
(The number to show can be varied with 'limit')	(The number to show can be varied with 'limit')
'''	'''
month = c.month or 'All'	month = c.month or 'All'
connection = model.Session.connection()	connection = model.Session.connection()
q = """	q = """
select department_id, sum(pageviews::int) views, sum(visits::int) visits	select department_id, sum(pageviews::int) views, sum(visits::int) visits
from ga_url	from ga_url
where department_id <> ''	where department_id <> ''
and package_id <> ''	and package_id <> ''
and url like '/dataset/%%'	and url like '/dataset/%%'
and period_name=%s	and period_name=%s
group by department_id order by views desc	group by department_id order by views desc
"""	"""
if limit:	if limit:
q = q + " limit %s;" % (limit)	q = q + " limit %s;" % (limit)

top_publishers = []	top_publishers = []
res = connection.execute(q, month)	res = connection.execute(q, month)
for row in res:	for row in res:
g = model.Group.get(row[0])	g = model.Group.get(row[0])
if g:	if g:
top_publishers.append((g, row[1], row[2]))	top_publishers.append((g, row[1], row[2]))
return top_publishers	return top_publishers


def _get_top_publishers_graph(limit=20):	def _get_top_publishers_graph(limit=20):
'''	'''
Returns a list of the top 20 publishers by dataset visits.	Returns a list of the top 20 publishers by dataset visits.
(The number to show can be varied with 'limit')	(The number to show can be varied with 'limit')
'''	'''
connection = model.Session.connection()	connection = model.Session.connection()
q = """	q = """
select department_id, sum(pageviews::int) views	select department_id, sum(pageviews::int) views
from ga_url	from ga_url
where department_id <> ''	where department_id <> ''
and package_id <> ''	and package_id <> ''
and url like '/dataset/%%'	and url like '/dataset/%%'
and period_name='All'	and period_name='All'
group by department_id order by views desc	group by department_id order by views desc
"""	"""
if limit:	if limit:
q = q + " limit %s;" % (limit)	q = q + " limit %s;" % (limit)

res = connection.execute(q)	res = connection.execute(q)
department_ids = [ row[0] for row in res ]	department_ids = [ row[0] for row in res ]

# Query for a history graph of these department ids	# Query for a history graph of these department ids
q = model.Session.query(	q = model.Session.query(
GA_Url.department_id,	GA_Url.department_id,
GA_Url.period_name,	GA_Url.period_name,
func.sum(cast(GA_Url.pageviews,sqlalchemy.types.INT)))\	func.sum(cast(GA_Url.pageviews,sqlalchemy.types.INT)))\
.filter( GA_Url.department_id.in_(department_ids) )\	.filter( GA_Url.department_id.in_(department_ids) )\
.filter( GA_Url.url.like('/dataset/%') )\	.filter( GA_Url.url.like('/dataset/%') )\
.filter( GA_Url.package_id!='' )\	.filter( GA_Url.package_id!='' )\
.group_by( GA_Url.department_id, GA_Url.period_name )	.group_by( GA_Url.department_id, GA_Url.period_name )
graph_dict = {}	graph_dict = {}
for dept_id,period_name,views in q:	for dept_id,period_name,views in q:
graph_dict[dept_id] = graph_dict.get( dept_id, {	graph_dict[dept_id] = graph_dict.get( dept_id, {
'name' : model.Group.get(dept_id).title,	'name' : model.Group.get(dept_id).title,
'raw' : {}	'raw' : {}
})	})
graph_dict[dept_id]['raw'][period_name] = views	graph_dict[dept_id]['raw'][period_name] = views
return [ graph_dict[id] for id in department_ids ]	return [ graph_dict[id] for id in department_ids ]


def _get_publishers():	def _get_publishers():
'''	'''
Returns a list of all publishers. Each item is a tuple:	Returns a list of all publishers. Each item is a tuple:
(name, title)	(name, title)
'''	'''
publishers = []	publishers = []
for pub in model.Session.query(model.Group).\	for pub in model.Session.query(model.Group).\
filter(model.Group.type=='organization').\	filter(model.Group.type=='organization').\
filter(model.Group.state=='active').\	filter(model.Group.state=='active').\
order_by(model.Group.name):	order_by(model.Group.name):
publishers.append((pub.name, pub.title))	publishers.append((pub.name, pub.title))
return publishers	return publishers

def _percent(num, total):	def _percent(num, total):
p = 100 * float(num)/float(total)	p = 100 * float(num)/float(total)
return "%.2f%%" % round(p, 2)	return "%.2f%%" % round(p, 2)

file:a/ckanext/ga_report/download_analytics.py -> file:b/ckanext/ga_report/download_analytics.py

import os	import os
import logging	import logging
import datetime	import datetime
import httplib	import httplib
	import urllib
import collections	import collections
import requests	import requests
import json	import json
	import re
from pylons import config	from pylons import config
from ga_model import _normalize_url	from ga_model import _normalize_url
import ga_model	import ga_model

#from ga_client import GA	#from ga_client import GA

log = logging.getLogger('ckanext.ga-report')	log = logging.getLogger('ckanext.ga-report')

FORMAT_MONTH = '%Y-%m'	FORMAT_MONTH = '%Y-%m'
MIN_VIEWS = 50	MIN_VIEWS = 50
MIN_VISITS = 20	MIN_VISITS = 20
MIN_DOWNLOADS = 10	MIN_DOWNLOADS = 10

class DownloadAnalytics(object):	class DownloadAnalytics(object):
'''Downloads and stores analytics info'''	'''Downloads and stores analytics info'''

def __init__(self, service=None, token=None, profile_id=None, delete_first=False,	def __init__(self, service=None, token=None, profile_id=None, delete_first=False,
skip_url_stats=False):	skip_url_stats=False):
self.period = config['ga-report.period']	self.period = config['ga-report.period']
self.service = service	self.service = service
self.profile_id = profile_id	self.profile_id = profile_id
self.delete_first = delete_first	self.delete_first = delete_first
self.skip_url_stats = skip_url_stats	self.skip_url_stats = skip_url_stats
self.token = token	self.token = token

def specific_month(self, date):	def specific_month(self, date):
import calendar	import calendar

first_of_this_month = datetime.datetime(date.year, date.month, 1)	first_of_this_month = datetime.datetime(date.year, date.month, 1)
_, last_day_of_month = calendar.monthrange(int(date.year), int(date.month))	_, last_day_of_month = calendar.monthrange(int(date.year), int(date.month))
last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month)	last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month)
# if this is the latest month, note that it is only up until today	# if this is the latest month, note that it is only up until today
now = datetime.datetime.now()	now = datetime.datetime.now()
if now.year == date.year and now.month == date.month:	if now.year == date.year and now.month == date.month:
last_day_of_month = now.day	last_day_of_month = now.day
last_of_this_month = now	last_of_this_month = now
periods = ((date.strftime(FORMAT_MONTH),	periods = ((date.strftime(FORMAT_MONTH),
last_day_of_month,	last_day_of_month,
first_of_this_month, last_of_this_month),)	first_of_this_month, last_of_this_month),)
self.download_and_store(periods)	self.download_and_store(periods)


def latest(self):	def latest(self):
if self.period == 'monthly':	if self.period == 'monthly':
# from first of this month to today	# from first of this month to today
now = datetime.datetime.now()	now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1)	first_of_this_month = datetime.datetime(now.year, now.month, 1)
periods = ((now.strftime(FORMAT_MONTH),	periods = ((now.strftime(FORMAT_MONTH),
now.day,	now.day,
first_of_this_month, now),)	first_of_this_month, now),)
else:	else:
raise NotImplementedError	raise NotImplementedError
self.download_and_store(periods)	self.download_and_store(periods)


def for_date(self, for_date):	def for_date(self, for_date):
assert isinstance(since_date, datetime.datetime)	assert isinstance(since_date, datetime.datetime)
periods = [] # (period_name, period_complete_day, start_date, end_date)	periods = [] # (period_name, period_complete_day, start_date, end_date)
if self.period == 'monthly':	if self.period == 'monthly':
first_of_the_months_until_now = []	first_of_the_months_until_now = []
year = for_date.year	year = for_date.year
month = for_date.month	month = for_date.month
now = datetime.datetime.now()	now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1)	first_of_this_month = datetime.datetime(now.year, now.month, 1)
while True:	while True:
first_of_the_month = datetime.datetime(year, month, 1)	first_of_the_month = datetime.datetime(year, month, 1)
if first_of_the_month == first_of_this_month:	if first_of_the_month == first_of_this_month:
periods.append((now.strftime(FORMAT_MONTH),	periods.append((now.strftime(FORMAT_MONTH),
now.day,	now.day,
first_of_this_month, now))	first_of_this_month, now))
break	break
elif first_of_the_month < first_of_this_month:	elif first_of_the_month < first_of_this_month:
in_the_next_month = first_of_the_month + datetime.timedelta(40)	in_the_next_month = first_of_the_month + datetime.timedelta(40)
last_of_the_month = datetime.datetime(in_the_next_month.year,	last_of_the_month = datetime.datetime(in_the_next_month.year,
in_the_next_month.month, 1)\	in_the_next_month.month, 1)\
- datetime.timedelta(1)	- datetime.timedelta(1)
periods.append((now.strftime(FORMAT_MONTH), 0,	periods.append((now.strftime(FORMAT_MONTH), 0,
first_of_the_month, last_of_the_month))	first_of_the_month, last_of_the_month))
else:	else:
# first_of_the_month has got to the future somehow	# first_of_the_month has got to the future somehow
break	break
month += 1	month += 1
if month > 12:	if month > 12:
year += 1	year += 1
month = 1	month = 1
else:	else:
raise NotImplementedError	raise NotImplementedError
self.download_and_store(periods)	self.download_and_store(periods)

@staticmethod	@staticmethod
def get_full_period_name(period_name, period_complete_day):	def get_full_period_name(period_name, period_complete_day):
if period_complete_day:	if period_complete_day:
return period_name + ' (up to %ith)' % period_complete_day	return period_name + ' (up to %ith)' % period_complete_day
else:	else:
return period_name	return period_name


def download_and_store(self, periods):	def download_and_store(self, periods):
for period_name, period_complete_day, start_date, end_date in periods:	for period_name, period_complete_day, start_date, end_date in periods:
log.info('Period "%s" (%s - %s)',	log.info('Period "%s" (%s - %s)',
self.get_full_period_name(period_name, period_complete_day),	self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y-%m-%d'),	start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))	end_date.strftime('%Y-%m-%d'))

if self.delete_first:	if self.delete_first:
log.info('Deleting existing Analytics for this period "%s"',	log.info('Deleting existing Analytics for this period "%s"',
period_name)	period_name)
ga_model.delete(period_name)	ga_model.delete(period_name)

if not self.skip_url_stats:	if not self.skip_url_stats:
# Clean out old url data before storing the new	# Clean out old url data before storing the new
ga_model.pre_update_url_stats(period_name)	ga_model.pre_update_url_stats(period_name)

accountName = config.get('googleanalytics.account')	accountName = config.get('googleanalytics.account')

log.info('Downloading analytics for dataset views')	log.info('Downloading analytics for dataset views')
data = self.download(start_date, end_date, '~^/dataset/[a-z0-9-_]+')	data = self.download(start_date, end_date, '~^/dataset/[a-z0-9-_]+')

log.info('Storing dataset views (%i rows)', len(data.get('url')))	log.info('Storing dataset views (%i rows)', len(data.get('url')))
self.store(period_name, period_complete_day, data, )	self.store(period_name, period_complete_day, data, )

log.info('Downloading analytics for publisher views')	log.info('Downloading analytics for publisher views')
data = self.download(start_date, end_date, '~^/organization/[a-z0-9-_]+')	data = self.download(start_date, end_date, '~^/organization/[a-z0-9-_]+')

log.info('Storing publisher views (%i rows)', len(data.get('url')))	log.info('Storing publisher views (%i rows)', len(data.get('url')))
self.store(period_name, period_complete_day, data,)	self.store(period_name, period_complete_day, data,)

# Make sure the All records are correct.	# Make sure the All records are correct.
ga_model.post_update_url_stats()	ga_model.post_update_url_stats()

log.info('Associating datasets with their publisher')	log.info('Associating datasets with their publisher')
ga_model.update_publisher_stats(period_name) # about 30 seconds.	ga_model.update_publisher_stats(period_name) # about 30 seconds.


log.info('Downloading and storing analytics for site-wide stats')	log.info('Downloading and storing analytics for site-wide stats')
self.sitewide_stats( period_name, period_complete_day )	self.sitewide_stats( period_name, period_complete_day )

log.info('Downloading and storing analytics for social networks')	log.info('Downloading and storing analytics for social networks')
self.update_social_info(period_name, start_date, end_date)	self.update_social_info(period_name, start_date, end_date)


def update_social_info(self, period_name, start_date, end_date):	def update_social_info(self, period_name, start_date, end_date):
start_date = start_date.strftime('%Y-%m-%d')	start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')	end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:hasSocialSourceReferral=~Yes$'	query = 'ga:hasSocialSourceReferral=~Yes$'
metrics = 'ga:entrances'	metrics = 'ga:entrances'
sort = '-ga:entrances'	sort = '-ga:entrances'

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict(ids='ga:' + self.profile_id,	args = dict(ids='ga:' + self.profile_id,
filters=query,	filters=query,
metrics=metrics,	metrics=metrics,
sort=sort,	sort=sort,
dimensions="ga:landingPagePath,ga:socialNetwork",	dimensions="ga:landingPagePath,ga:socialNetwork",
max_results=10000)	max_results=10000)

args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])


data = collections.defaultdict(list)	data = collections.defaultdict(list)
rows = results.get('rows',[])	rows = results.get('rows',[])
for row in rows:	for row in rows:
url = row[0]	url = row[0]
data[url].append( (row[1], int(row[2]),) )	data[url].append( (row[1], int(row[2]),) )
ga_model.update_social(period_name, data)	ga_model.update_social(period_name, data)


def download(self, start_date, end_date, path=None):	def download(self, start_date, end_date, path=None):
'''Get data from GA for a given time period'''	'''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d')	start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')	end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:pagePath=%s$' % path	query = 'ga:pagePath=%s$' % path
metrics = 'ga:pageviews, ga:visits'	metrics = 'ga:pageviews, ga:visits'
sort = '-ga:pageviews'	sort = '-ga:pageviews'

# Supported query params at	# Supported query params at
# https://developers.google.com/analytics/devguides/reporting/core/v3/reference	# https://developers.google.com/analytics/devguides/reporting/core/v3/reference
# https://ga-dev-tools.appspot.com/explorer/	# https://ga-dev-tools.appspot.com/explorer/
try:	try:
args = {}	args = {}
args["sort"] = "-ga:pageviews"	args["sort"] = "-ga:pageviews"
args["max-results"] = 100000	args["max-results"] = 100000
args["dimensions"] = "ga:pagePath"	args["dimensions"] = "ga:pagePath"
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["metrics"] = metrics	args["metrics"] = metrics
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id
args["filters"] = query	args["filters"] = query
args["alt"] = "json"	args["alt"] = "json"
print args	print args
results = self._get_json(args)	results = self._get_json(args)

except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
return dict(url=[])	return dict(url=[])

packages = []	packages = []
log.info("There are %d results" % results['totalResults'])	log.info("There are %d results" % results['totalResults'])
if results['totalResults'] > 0:	if results['totalResults'] > 0:
for entry in results.get('rows'):	for entry in results.get('rows'):
(loc,pageviews,visits) = entry	(loc,pageviews,visits) = entry
#url = _normalize_url('http:/' + loc) # strips off domain e.g. www.data.gov.uk or data.gov.uk	#url = _normalize_url('http:/' + loc) # strips off domain e.g. www.data.gov.uk or data.gov.uk
url = loc	url = loc
#print url	#print url
if not url.startswith('/dataset/') and not url.startswith('/organization/'):	if not url.startswith('/dataset/') and not url.startswith('/organization/'):
# filter out strays like:	# filter out strays like:
# /data/user/login?came_from=http://data.gov.uk/dataset/os-code-point-open	# /data/user/login?came_from=http://data.gov.uk/dataset/os-code-point-open
# /403.html?page=/about&from=http://data.gov.uk/publisher/planning-inspectorate	# /403.html?page=/about&from=http://data.gov.uk/publisher/planning-inspectorate
continue	continue
packages.append( (url, pageviews, visits,) ) # Temporary hack	packages.append( (url, pageviews, visits,) ) # Temporary hack
return dict(url=packages)	return dict(url=packages)

def store(self, period_name, period_complete_day, data):	def store(self, period_name, period_complete_day, data):
if 'url' in data:	if 'url' in data:
ga_model.update_url_stats(period_name, period_complete_day, data['url'])	ga_model.update_url_stats(period_name, period_complete_day, data['url'])

def sitewide_stats(self, period_name, period_complete_day):	def sitewide_stats(self, period_name, period_complete_day):
import calendar	import calendar
year, month = period_name.split('-')	year, month = period_name.split('-')
_, last_day_of_month = calendar.monthrange(int(year), int(month))	_, last_day_of_month = calendar.monthrange(int(year), int(month))

start_date = '%s-01' % period_name	start_date = '%s-01' % period_name
end_date = '%s-%s' % (period_name, last_day_of_month)	end_date = '%s-%s' % (period_name, last_day_of_month)
funcs = ['_totals_stats', '_social_stats', '_os_stats',	funcs = ['_totals_stats', '_social_stats', '_os_stats',
'_locale_stats', '_browser_stats', '_mobile_stats', '_download_stats']	'_locale_stats', '_browser_stats', '_mobile_stats', '_download_stats']
for f in funcs:	for f in funcs:
log.info('Downloading analytics for %s' % f.split('_')[1])	log.info('Downloading analytics for %s' % f.split('_')[1])
getattr(self, f)(start_date, end_date, period_name, period_complete_day)	getattr(self, f)(start_date, end_date, period_name, period_complete_day)

def _get_results(result_data, f):	def _get_results(result_data, f):
data = {}	data = {}
for result in result_data:	for result in result_data:
key = f(result)	key = f(result)
data[key] = data.get(key,0) + result[1]	data[key] = data.get(key,0) + result[1]
return data	return data

def _get_json(self, params, prev_fail=False):	def _get_json(self, params, prev_fail=False):
ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))	ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:	if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \	print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'	'Google Analytics token file under key: googleanalytics.token.filepath'
return	return

log.info("Trying to refresh our OAuth token")	log.info("Trying to refresh our OAuth token")
try:	try:
from ga_auth import init_service	from ga_auth import init_service
self.token, svc = init_service(ga_token_filepath, None)	self.token, svc = init_service(ga_token_filepath, None)
log.info("OAuth token refreshed")	log.info("OAuth token refreshed")
except Exception, auth_exception:	except Exception, auth_exception:
log.error("Oauth refresh failed")	log.error("Oauth refresh failed")
log.exception(auth_exception)	log.exception(auth_exception)
return	return

try:	try:
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}
r = requests.get("https://www.googleapis.com/analytics/v3/data/ga", params=params, headers=headers)	r = requests.get("https://www.googleapis.com/analytics/v3/data/ga", params=params, headers=headers)
if r.status_code != 200:	if r.status_code != 200:
log.info("STATUS: %s" % (r.status_code,))	log.info("STATUS: %s" % (r.status_code,))
log.info("CONTENT: %s" % (r.content,))	log.info("CONTENT: %s" % (r.content,))
raise Exception("Request with params: %s failed" % params)	raise Exception("Request with params: %s failed" % params)

return json.loads(r.content)	return json.loads(r.content)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)

return dict(url=[])	return dict(url=[])

def _totals_stats(self, start_date, end_date, period_name, period_complete_day):	def _totals_stats(self, start_date, end_date, period_name, period_complete_day):
""" Fetches distinct totals, total pageviews etc """	""" Fetches distinct totals, total pageviews etc """
try:	try:
args = {}	args = {}
args["max-results"] = 100000	args["max-results"] = 100000
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id

args["metrics"] = "ga:pageviews"	args["metrics"] = "ga:pageviews"
args["sort"] = "-ga:pageviews"	args["sort"] = "-ga:pageviews"
args["alt"] = "json"	args["alt"] = "json"

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]},	ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]},
period_complete_day)	period_complete_day)

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = {}	args = {}
args["max-results"] = 100000	args["max-results"] = 100000
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id

args["metrics"] = "ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visits"	args["metrics"] = "ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visits"
args["alt"] = "json"	args["alt"] = "json"

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
data = {	data = {
'Pages per visit': result_data[0][0],	'Pages per visit': result_data[0][0],
'Average time on site': result_data[0][1],	'Average time on site': result_data[0][1],
'New visits': result_data[0][2],	'New visits': result_data[0][2],
'Total visits': result_data[0][3],	'Total visits': result_data[0][3],
}	}
ga_model.update_sitewide_stats(period_name, "Totals", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Totals", data, period_complete_day)

# Bounces from / or another configurable page.	# Bounces from / or another configurable page.
path = '/' #% (config.get('googleanalytics.account'), config.get('ga-report.bounce_url', '/'))	path = '/' #% (config.get('googleanalytics.account'), config.get('ga-report.bounce_url', '/'))

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = {}	args = {}
args["max-results"] = 100000	args["max-results"] = 100000
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id

args["filters"] = 'ga:pagePath==%s' % (path,)	args["filters"] = 'ga:pagePath==%s' % (path,)
args["dimensions"] = 'ga:pagePath'	args["dimensions"] = 'ga:pagePath'
args["metrics"] = "ga:visitBounceRate"	args["metrics"] = "ga:visitBounceRate"
args["alt"] = "json"	args["alt"] = "json"

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
if not result_data or len(result_data) != 1:	if not result_data or len(result_data) != 1:
log.error('Could not pinpoint the bounces for path: %s. Got results: %r',	log.error('Could not pinpoint the bounces for path: %s. Got results: %r',
path, result_data)	path, result_data)
return	return
results = result_data[0]	results = result_data[0]
bounces = float(results[1])	bounces = float(results[1])
# visitBounceRate is already a %	# visitBounceRate is already a %
log.info('Google reports visitBounceRate as %s', bounces)	log.info('Google reports visitBounceRate as %s', bounces)
ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': float(bounces)},	ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': float(bounces)},
period_complete_day)	period_complete_day)


def _locale_stats(self, start_date, end_date, period_name, period_complete_day):	def _locale_stats(self, start_date, end_date, period_name, period_complete_day):
""" Fetches stats about language and country """	""" Fetches stats about language and country """

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = {}	args = {}
args["max-results"] = 100000	args["max-results"] = 100000
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id

args["dimensions"] = "ga:language,ga:country"	args["dimensions"] = "ga:language,ga:country"
args["metrics"] = "ga:pageviews"	args["metrics"] = "ga:pageviews"
args["sort"] = "-ga:pageviews"	args["sort"] = "-ga:pageviews"
args["alt"] = "json"	args["alt"] = "json"

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[0]] = data.get(result[0], 0) + int(result[2])	data[result[0]] = data.get(result[0], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Languages", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Languages", data, period_complete_day)

data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[1]] = data.get(result[1], 0) + int(result[2])	data[result[1]] = data.get(result[1], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Country", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Country", data, period_complete_day)


def _download_stats(self, start_date, end_date, period_name, period_complete_day):	def _download_stats(self, start_date, end_date, period_name, period_complete_day):
""" Fetches stats about data downloads """	""" Fetches stats about data downloads """
import ckan.model as model	import ckan.model as model

data = {}	data = {}

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = {}	args = {}
args["max-results"] = 100000	args["max-results"] = 100000
args["start-date"] = start_date	args["start-date"] = start_date
args["end-date"] = end_date	args["end-date"] = end_date
args["ids"] = "ga:" + self.profile_id	args["ids"] = "ga:" + self.profile_id

args["filters"] = 'ga:eventAction==download'	args["filters"] = 'ga:eventAction==Download'
args["dimensions"] = "ga:eventLabel"	args["dimensions"] = "ga:eventLabel"
args["metrics"] = "ga:totalEvents"	args["metrics"] = "ga:totalEvents"
args["alt"] = "json"	args["alt"] = "json"

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
if not result_data:	if not result_data:
# We may not have data for this time period, so we need to bail	# We may not have data for this time period, so we need to bail
# early.	# early.
log.info("There is no download data for this time period")	log.info("There is no download data for this time period")
return	return

def process_result_data(result_data, cached=False):	def process_result_data(result_data, cached=False):
progress_total = len(result_data)	progress_total = len(result_data)
progress_count = 0	progress_count = 0
resources_not_matched = []	resources_not_matched = []
for result in result_data:	for result in result_data:
progress_count += 1	progress_count += 1
if progress_count % 100 == 0:	if progress_count % 100 == 0:
log.debug('.. %d/%d done so far', progress_count, progress_total)	log.debug('.. %d/%d done so far', progress_count, progress_total)

url = result[0].strip()	url = urllib.unquote(result[0].strip())

# Get package id associated with the resource that has this URL.	# Get package id associated with the resource that has this URL.
q = model.Session.query(model.Resource)	q = model.Session.query(model.Resource)
if cached:	if cached:
r = q.filter(model.Resource.cache_url.like("%s%%" % url)).first()	r = q.filter(model.Resource.cache_url.like("%s%%" % url)).first()
else:	else:
r = q.filter(model.Resource.url.like("%s%%" % url)).first()	r = q.filter(model.Resource.url.like("%s%%" % url)).first()

	# new style internal download links
	if re.search('(?:/resource/)(.*)(?:/download/)',url):
	resource_id = re.search('(?:/resource/)(.*)(?:/download/)',url)
	r = q.filter(model.Resource.id.like("%s%%" % resource_id.group(1))).first()

package_name = r.resource_group.package.name if r else ""	package_name = r.resource_group.package.name if r else ""


if package_name:	if package_name:
data[package_name] = data.get(package_name, 0) + int(result[1])	data[package_name] = data.get(package_name, 0) + int(result[1])
else:	else:
resources_not_matched.append(url)	resources_not_matched.append(url)
continue	continue
if resources_not_matched:	if resources_not_matched:
log.debug('Could not match %i or %i resource URLs to datasets. e.g. %r',	log.debug('Could not match %i or %i resource URLs to datasets. e.g. %r',
len(resources_not_matched), progress_total, resources_not_matched[:3])	len(resources_not_matched), progress_total, resources_not_matched[:3])

log.info('Associating downloads of resource URLs with their respective datasets')	log.info('Associating downloads of resource URLs with their respective datasets')
process_result_data(results.get('rows'))	process_result_data(results.get('rows'))

try:	'''try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict( ids='ga:' + self.profile_id,	args = dict( ids='ga:' + self.profile_id,
filters='ga:eventAction==download-cache',	filters='ga:eventAction==download-cache',
metrics='ga:totalEvents',	metrics='ga:totalEvents',
sort='-ga:totalEvents',	sort='-ga:totalEvents',
dimensions="ga:eventLabel",	dimensions="ga:eventLabel",
max_results=10000)	max_results=10000)
args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

log.info('Associating downloads of cache resource URLs with their respective datasets')	log.info('Associating downloads of cache resource URLs with their respective datasets')
process_result_data(results.get('rows'), cached=False)	process_result_data(results.get('rows'), cached=False)'''

self._filter_out_long_tail(data, MIN_DOWNLOADS)	self._filter_out_long_tail(data, MIN_DOWNLOADS)
ga_model.update_sitewide_stats(period_name, "Downloads", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Downloads", data, period_complete_day)

def _social_stats(self, start_date, end_date, period_name, period_complete_day):	def _social_stats(self, start_date, end_date, period_name, period_complete_day):
""" Finds out which social sites people are referred from """	""" Finds out which social sites people are referred from """

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict( ids='ga:' + self.profile_id,	args = dict( ids='ga:' + self.profile_id,
metrics='ga:pageviews',	metrics='ga:pageviews',
sort='-ga:pageviews',	sort='-ga:pageviews',
dimensions="ga:socialNetwork,ga:referralPath",	dimensions="ga:socialNetwork,ga:referralPath",
max_results=10000)	max_results=10000)
args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
data = {}	data = {}
for result in result_data:	for result in result_data:
if not result[0] == '(not set)':	if not result[0] == '(not set)':
data[result[0]] = data.get(result[0], 0) + int(result[2])	data[result[0]] = data.get(result[0], 0) + int(result[2])
self._filter_out_long_tail(data, 3)	self._filter_out_long_tail(data, 3)
ga_model.update_sitewide_stats(period_name, "Social sources", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Social sources", data, period_complete_day)


def _os_stats(self, start_date, end_date, period_name, period_complete_day):	def _os_stats(self, start_date, end_date, period_name, period_complete_day):
""" Operating system stats """	""" Operating system stats """
try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict( ids='ga:' + self.profile_id,	args = dict( ids='ga:' + self.profile_id,
metrics='ga:pageviews',	metrics='ga:pageviews',
sort='-ga:pageviews',	sort='-ga:pageviews',
dimensions="ga:operatingSystem,ga:operatingSystemVersion",	dimensions="ga:operatingSystem,ga:operatingSystemVersion",
max_results=10000)	max_results=10000)
args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])

result_data = results.get('rows')	result_data = results.get('rows')
data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[0]] = data.get(result[0], 0) + int(result[2])	data[result[0]] = data.get(result[0], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Operating Systems", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Operating Systems", data, period_complete_day)

data = {}	data = {}
for result in result_data:	for result in result_data:
if int(result[2]) >= MIN_VIEWS:	if int(result[2]) >= MIN_VIEWS:
key = "%s %s" % (result[0],result[1])	key = "%s %s" % (result[0],result[1])
data[key] = result[2]	data[key] = result[2]
ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data, period_complete_day)


def _browser_stats(self, start_date, end_date, period_name, period_complete_day):	def _browser_stats(self, start_date, end_date, period_name, period_complete_day):
""" Information about browsers and browser versions """	""" Information about browsers and browser versions """

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict( ids='ga:' + self.profile_id,	args = dict( ids='ga:' + self.profile_id,
metrics='ga:pageviews',	metrics='ga:pageviews',
sort='-ga:pageviews',	sort='-ga:pageviews',
dimensions="ga:browser,ga:browserVersion",	dimensions="ga:browser,ga:browserVersion",
max_results=10000)	max_results=10000)

args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])


result_data = results.get('rows')	result_data = results.get('rows')
# e.g. [u'Firefox', u'19.0', u'20']	# e.g. [u'Firefox', u'19.0', u'20']

data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[0]] = data.get(result[0], 0) + int(result[2])	data[result[0]] = data.get(result[0], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Browsers", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Browsers", data, period_complete_day)

data = {}	data = {}
for result in result_data:	for result in result_data:
key = "%s %s" % (result[0], self._filter_browser_version(result[0], result[1]))	key = "%s %s" % (result[0], self._filter_browser_version(result[0], result[1]))
data[key] = data.get(key, 0) + int(result[2])	data[key] = data.get(key, 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Browser versions", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Browser versions", data, period_complete_day)

@classmethod	@classmethod
def _filter_browser_version(cls, browser, version_str):	def _filter_browser_version(cls, browser, version_str):
'''	'''
Simplifies a browser version string if it is detailed.	Simplifies a browser version string if it is detailed.
i.e. groups together Firefox 3.5.1 and 3.5.2 to be just 3.	i.e. groups together Firefox 3.5.1 and 3.5.2 to be just 3.
This is helpful when viewing stats and good to protect privacy.	This is helpful when viewing stats and good to protect privacy.
'''	'''
ver = version_str	ver = version_str
parts = ver.split('.')	parts = ver.split('.')
if len(parts) > 1:	if len(parts) > 1:
if parts[1][0] == '0':	if parts[1][0] == '0':
ver = parts[0]	ver = parts[0]
else:	else:
ver = "%s" % (parts[0])	ver = "%s" % (parts[0])
# Special case complex version nums	# Special case complex version nums
if browser in ['Safari', 'Android Browser']:	if browser in ['Safari', 'Android Browser']:
ver = parts[0]	ver = parts[0]
if len(ver) > 2:	if len(ver) > 2:
num_hidden_digits = len(ver) - 2	num_hidden_digits = len(ver) - 2
ver = ver[0] + ver[1] + 'X' * num_hidden_digits	ver = ver[0] + ver[1] + 'X' * num_hidden_digits
return ver	return ver

def _mobile_stats(self, start_date, end_date, period_name, period_complete_day):	def _mobile_stats(self, start_date, end_date, period_name, period_complete_day):
""" Info about mobile devices """	""" Info about mobile devices """

try:	try:
# Because of issues of invalid responses, we are going to make these requests	# Because of issues of invalid responses, we are going to make these requests
# ourselves.	# ourselves.
headers = {'authorization': 'Bearer ' + self.token}	headers = {'authorization': 'Bearer ' + self.token}

args = dict( ids='ga:' + self.profile_id,	args = dict( ids='ga:' + self.profile_id,
metrics='ga:pageviews',	metrics='ga:pageviews',
sort='-ga:pageviews',	sort='-ga:pageviews',
dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",	dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",
max_results=10000)	max_results=10000)
args['start-date'] = start_date	args['start-date'] = start_date
args['end-date'] = end_date	args['end-date'] = end_date

results = self._get_json(args)	results = self._get_json(args)
except Exception, e:	except Exception, e:
log.exception(e)	log.exception(e)
results = dict(url=[])	results = dict(url=[])


result_data = results.get('rows')	result_data = results.get('rows')
data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[0]] = data.get(result[0], 0) + int(result[2])	data[result[0]] = data.get(result[0], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Mobile brands", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Mobile brands", data, period_complete_day)

data = {}	data = {}
for result in result_data:	for result in result_data:
data[result[1]] = data.get(result[1], 0) + int(result[2])	data[result[1]] = data.get(result[1], 0) + int(result[2])
self._filter_out_long_tail(data, MIN_VIEWS)	self._filter_out_long_tail(data, MIN_VIEWS)
ga_model.update_sitewide_stats(period_name, "Mobile devices", data, period_complete_day)	ga_model.update_sitewide_stats(period_name, "Mobile devices", data, period_complete_day)

@classmethod	@classmethod
def _filter_out_long_tail(cls, data, threshold=10):	def _filter_out_long_tail(cls, data, threshold=10):
'''	'''
Given data which is a frequency distribution, filter out	Given data which is a frequency distribution, filter out
results which are below a threshold count. This is good to protect	results which are below a threshold count. This is good to protect
privacy.	privacy.
'''	'''
for key, value in data.items():	for key, value in data.items():
if value < threshold:	if value < threshold:
del data[key]	del data[key]

file:a/ckanext/ga_report/helpers.py -> file:b/ckanext/ga_report/helpers.py

import logging	import logging
import operator	import operator

import ckan.lib.base as base	import ckan.lib.base as base
import ckan.model as model	import ckan.model as model
from ckan.logic import get_action	from ckan.logic import get_action

from ckanext.ga_report.ga_model import GA_Url, GA_Publisher	from ckanext.ga_report.ga_model import GA_Url, GA_Publisher
from ckanext.ga_report.controller import _get_publishers	from ckanext.ga_report.controller import _get_publishers
_log = logging.getLogger(__name__)	_log = logging.getLogger(__name__)

def popular_datasets(count=10):	def popular_datasets(count=10):
import random	import random

publisher = None	publisher = None
publishers = _get_publishers(30)	publishers = _get_publishers(30)
total = len(publishers)	total = len(publishers)
while not publisher or not datasets:	while not publisher or not datasets:
rand = random.randrange(0, total)	rand = random.randrange(0, total)
publisher = publishers[rand][0]	publisher = publishers[rand][0]
if not publisher.state == 'active':	if not publisher.state == 'active':
publisher = None	publisher = None
continue	continue
datasets = _datasets_for_publisher(publisher, 10)[:count]	datasets = _datasets_for_publisher(publisher, 10)[:count]

ctx = {	ctx = {
'datasets': datasets,	'datasets': datasets,
'publisher': publisher	'publisher': publisher
}	}
return base.render_snippet('ga_report/ga_popular_datasets.html', **ctx)	return base.render_snippet('ga_report/ga_popular_datasets.html', **ctx)

def single_popular_dataset(top=20):	def single_popular_dataset(top=20):
'''Returns a random dataset from the most popular ones.	'''Returns a random dataset from the most popular ones.

:param top: the number of top datasets to select from	:param top: the number of top datasets to select from
'''	'''
import random	import random

top_datasets = model.Session.query(GA_Url).\	top_datasets = model.Session.query(GA_Url).\
filter(GA_Url.url.like('/dataset/%')).\	filter(GA_Url.url.like('/dataset/%')).\
order_by('ga_url.pageviews::int desc')	order_by('ga_url.pageviews::int desc')
num_top_datasets = top_datasets.count()	num_top_datasets = top_datasets.count()

dataset = None	dataset = None
if num_top_datasets:	if num_top_datasets:
count = 0	count = 0
while not dataset:	while not dataset:
rand = random.randrange(0, min(top, num_top_datasets))	rand = random.randrange(0, min(top, num_top_datasets))
ga_url = top_datasets[rand]	ga_url = top_datasets[rand]
dataset = model.Package.get(ga_url.url[len('/dataset/'):])	dataset = model.Package.get(ga_url.url[len('/dataset/'):])
if dataset and not dataset.state == 'active':	if dataset and not dataset.state == 'active':
dataset = None	dataset = None
# When testing, it is possible that top datasets are not available	# When testing, it is possible that top datasets are not available
# so only go round this loop a few times before falling back on	# so only go round this loop a few times before falling back on
# a random dataset.	# a random dataset.
count += 1	count += 1
if count > 10:	if count > 10:
break	break
if not dataset:	if not dataset:
# fallback	# fallback
dataset = model.Session.query(model.Package)\	dataset = model.Session.query(model.Package)\
.filter_by(state='active').first()	.filter_by(state='active').first()
if not dataset:	if not dataset:
return None	return None
dataset_dict = get_action('package_show')({'model': model,	dataset_dict = get_action('package_show')({'model': model,
'session': model.Session,	'session': model.Session,
'validate': False},	'validate': False},
{'id':dataset.id})	{'id':dataset.id})
return dataset_dict	return dataset_dict

def single_popular_dataset_html(top=20):	def single_popular_dataset_html(top=20):
dataset_dict = single_popular_dataset(top)	dataset_dict = single_popular_dataset(top)
groups = package.get('groups', [])	groups = package.get('groups', [])
publishers = [ g for g in groups if g.get('type') == 'organization' ]	publishers = [ g for g in groups if g.get('type') == 'organization' ]
publisher = publishers[0] if publishers else {'name':'', 'title': ''}	publisher = publishers[0] if publishers else {'name':'', 'title': ''}
context = {	context = {
'dataset': dataset_dict,	'dataset': dataset_dict,
'publisher': publisher_dict	'publisher': publisher_dict
}	}
return base.render_snippet('ga_report/ga_popular_single.html', **context)	return base.render_snippet('ga_report/ga_popular_single.html', **context)


def most_popular_datasets(publisher, count=20, preview_image=None):	def most_popular_datasets(publisher, count=20, preview_image=None):

if not publisher:	if not publisher:
_log.error("No valid publisher passed to 'most_popular_datasets'")	_log.error("No valid publisher passed to 'most_popular_datasets'")
return ""	return ""

results = _datasets_for_publisher(publisher, count)	results = _datasets_for_publisher(publisher, count)

ctx = {	ctx = {
'dataset_count': len(results),	'dataset_count': len(results),
'datasets': results,	'datasets': results,

'publisher': publisher,	'publisher': publisher,
'preview_image': preview_image	'preview_image': preview_image
}	}

return base.render_snippet('ga_report/publisher/popular.html', **ctx)	return base.render_snippet('ga_report/publisher/popular.html', **ctx)

def _datasets_for_publisher(publisher, count):	def _datasets_for_publisher(publisher, count):
datasets = {}	datasets = {}
entries = model.Session.query(GA_Url).\	entries = model.Session.query(GA_Url).\
filter(GA_Url.department_id==publisher.name).\	filter(GA_Url.department_id==publisher.name).\
filter(GA_Url.url.like('/dataset/%')).\	filter(GA_Url.url.like('/dataset/%')).\
order_by('ga_url.pageviews::int desc').all()	order_by('ga_url.pageviews::int desc').all()
for entry in entries:	for entry in entries:
if len(datasets) < count:	if len(datasets) < count:
p = model.Package.get(entry.url[len('/dataset/'):])	p = model.Package.get(entry.url[len('/dataset/'):])

if not p:	if not p:
_log.warning("Could not find Package for {url}".format(url=entry.url))	_log.warning("Could not find Package for {url}".format(url=entry.url))
continue	continue

if not p.state == 'active':	if not p.state == 'active':
_log.warning("Package {0} is not active, it is {1}".format(p.name, p.state))	_log.warning("Package {0} is not active, it is {1}".format(p.name, p.state))
continue	continue

	if not p.private == False:
	_log.warning("Package {0} is private {1}".format(p.name, p.state))
	continue

if not p in datasets:	if not p in datasets:
datasets[p] = {'views':0, 'visits': 0}	datasets[p] = {'views':0, 'visits': 0}

datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)	datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visits)	datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visits)

results = []	results = []
for k, v in datasets.iteritems():	for k, v in datasets.iteritems():
results.append((k,v['views'],v['visits']))	results.append((k,v['views'],v['visits']))

return sorted(results, key=operator.itemgetter(1), reverse=True)	return sorted(results, key=operator.itemgetter(1), reverse=True)

def month_option_title(month_iso, months, day):	def month_option_title(month_iso, months, day):
month_isos = [ iso_code for (iso_code,name) in months ]	month_isos = [ iso_code for (iso_code,name) in months ]
try:	try:
index = month_isos.index(month_iso)	index = month_isos.index(month_iso)
except ValueError:	except ValueError:
_log.error('Month "%s" not found in list of months.' % month_iso)	_log.error('Month "%s" not found in list of months.' % month_iso)
return month_iso	return month_iso
month_name = months[index][1]	month_name = months[index][1]
if index==0:	if index==0:
return month_name + (' (up to %s)'%day)	return month_name + (' (up to %s)'%day)
return month_name	return month_name

file:a/ckanext/ga_report/plugin.py -> file:b/ckanext/ga_report/plugin.py

import logging	import logging
import ckan.lib.helpers as h	import ckan.lib.helpers as h
import ckan.plugins as p	import ckan.plugins as p
from ckan.plugins import implements, toolkit	from ckan.plugins import implements, toolkit

from ckanext.ga_report.helpers import (most_popular_datasets,	from ckanext.ga_report.helpers import (most_popular_datasets,
popular_datasets,	popular_datasets,
single_popular_dataset,	single_popular_dataset,
month_option_title)	month_option_title)

log = logging.getLogger('ckanext.ga-report')	log = logging.getLogger('ckanext.ga-report')

class GAReportPlugin(p.SingletonPlugin):	class GAReportPlugin(p.SingletonPlugin):
implements(p.IConfigurer, inherit=True)	implements(p.IConfigurer, inherit=True)
implements(p.IRoutes, inherit=True)	implements(p.IRoutes, inherit=True)
implements(p.ITemplateHelpers, inherit=True)	implements(p.ITemplateHelpers, inherit=True)

def update_config(self, config):	def update_config(self, config):
toolkit.add_template_directory(config, 'templates')	toolkit.add_template_directory(config, 'templates')
toolkit.add_public_directory(config, 'public')	toolkit.add_public_directory(config, 'public')

def get_helpers(self):	def get_helpers(self):
"""	"""
A dictionary of extra helpers that will be available to provide	A dictionary of extra helpers that will be available to provide
ga report info to templates.	ga report info to templates.
"""	"""
return {	return {
'ga_report_installed': lambda: True,	'ga_report_installed': lambda: True,
'popular_datasets': popular_datasets,	'popular_datasets': popular_datasets,
'most_popular_datasets': most_popular_datasets,	'most_popular_datasets': most_popular_datasets,
'single_popular_dataset': single_popular_dataset,	'single_popular_dataset': single_popular_dataset,
'month_option_title': month_option_title	'month_option_title': month_option_title
}	}

def after_map(self, map):	def after_map(self, map):
# GaReport	# GaReport
map.connect(	map.connect(
'/site-usage',	'/site-usage',
controller='ckanext.ga_report.controller:GaReport',	controller='ckanext.ga_report.controller:GaReport',
action='index'	action='index'
)	)
map.connect(	map.connect(
'/site-usage_{month}.csv',	'/site-usage_{month}.csv',
controller='ckanext.ga_report.controller:GaReport',	controller='ckanext.ga_report.controller:GaReport',
action='csv'	action='csv'
)	)
map.connect(	map.connect(
'/site-usage/downloads',	'/site-usage/downloads',
controller='ckanext.ga_report.controller:GaReport',	controller='ckanext.ga_report.controller:GaReport',
action='downloads'	action='downloads'
)	)
map.connect(	map.connect(
'/site-usage/downloads_{month}.csv',	'/site-usage/downloads_{month}.csv',
controller='ckanext.ga_report.controller:GaReport',	controller='ckanext.ga_report.controller:GaReport',
action='csv_downloads'	action='csv_downloads'
)	)

# GaDatasetReport	# GaDatasetReport
map.connect(	map.connect(
'/site-usage/publisher',	'/site-usage/publisher',
controller='ckanext.ga_report.controller:GaDatasetReport',	controller='ckanext.ga_report.controller:GaDatasetReport',
action='publishers'	action='publishers'
)	)
map.connect(	map.connect(
'/site-usage/publishers_{month}.csv',	'/site-usage/publishers_{month}.csv',
controller='ckanext.ga_report.controller:GaDatasetReport',	controller='ckanext.ga_report.controller:GaDatasetReport',
action='publisher_csv'	action='publisher_csv'
)	)
map.connect(	map.connect(
'/site-usagesetsets_{id}_{month}.csv',	'/site-usage/dataset/datasets_{id}_{month}.csv',
controller='ckanext.ga_report.controller:GaDatasetReport',	controller='ckanext.ga_report.controller:GaDatasetReport',
action='dataset_csv'	action='dataset_csv'
)	)
map.connect(	map.connect(
'/site-usageset',	'/site-usage/dataset',
controller='ckanext.ga_report.controller:GaDatasetReport',	controller='ckanext.ga_report.controller:GaDatasetReport',
action='read'	action='read'
)	)
map.connect(	map.connect(
'/site-usageset/{id}',	'/site-usage/dataset/{id}',
controller='ckanext.ga_report.controller:GaDatasetReport',	controller='ckanext.ga_report.controller:GaDatasetReport',
action='read_publisher'	action='read_publisher'
)	)
return map	return map

file:a/ckanext/ga_report/templates/ga_report/publisher/read.html -> file:b/ckanext/ga_report/templates/ga_report/publisher/read.html

<html xmlns:py="http://genshi.edgewall.org/"	<html xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"	xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:xi="http://www.w3.org/2001/XInclude"	xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">	py:strip="">

<xi:include href="../ga_util.html" />	<xi:include href="../ga_util.html" />

<py:def function="page_title">Usage by Dataset</py:def>	<py:def function="page_title">Usage by Dataset</py:def>

<py:def function="optional_head">	<py:def function="optional_head">
<link rel="stylesheet" type="text/css" href="/scripts/vendor/rickshaw.min.css"/>	<link rel="stylesheet" type="text/css" href="/scripts/vendor/rickshaw.min.css"/>
<link rel="stylesheet" type="text/css" href="/css/ga_report.css?1"/>	<link rel="stylesheet" type="text/css" href="/css/ga_report.css?1"/>
<script type="text/javascript" src="/scripts/modernizr-2.6.2.custom.js"></script>	<script type="text/javascript" src="/scripts/modernizr-2.6.2.custom.js"></script>
<script type="text/javascript" src="/scripts/ckanext_ga_reports.js?1"></script>	<script type="text/javascript" src="/scripts/ckanext_ga_reports.js?1"></script>
<script type="text/javascript" src="/scripts/vendor/jquery.sparkline.modified.js"></script>	<script type="text/javascript" src="/scripts/vendor/jquery.sparkline.modified.js"></script>
<script type="text/javascript" src="/scripts/rickshaw_ie7_shim.js"></script>	<script type="text/javascript" src="/scripts/rickshaw_ie7_shim.js"></script>
<script type="text/javascript" src="/scripts/vendor/d3.v2.js"></script>	<script type="text/javascript" src="/scripts/vendor/d3.v2.js"></script>
<script type="text/javascript" src="/scripts/vendor/d3.layout.min.js"></script>	<script type="text/javascript" src="/scripts/vendor/d3.layout.min.js"></script>
<script type="text/javascript" src="/scripts/vendor/rickshaw.min.js"></script>	<script type="text/javascript" src="/scripts/vendor/rickshaw.min.js"></script>
</py:def>	</py:def>


<py:match path="breadcrumbs">	<py:match path="breadcrumbs">
<li><a href="/site-usage">Site Analytics</a></li>	<li><a href="/site-usage">Site Analytics</a></li>
<py:if test="c.publisher">	<py:if test="c.publisher">
<li><a href="/site-usage/publisher">Publishers</a></li>	<li><a href="/site-usage/publisher">Publishers</a></li>
<li py:if="c.publisher"><a href="/site-usage/publisher/${c.publisher.name}">${c.publisher.title}</a></li>	<li py:if="c.publisher"><a href="/site-usage/publisher/${c.publisher.name}">${c.publisher.title}</a></li>
</py:if>	</py:if>
<py:if test="not c.publisher">	<py:if test="not c.publisher">
<li><a href="${request.url}">Usage By Dataset</a></li>	<li><a href="${request.url}">Usage By Dataset</a></li>
</py:if>	</py:if>
</py:match>	</py:match>

<div py:match="content">	<div py:match="content">

<py:with vars="download_link=h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport',action='dataset_csv',id=c.publisher_name or 'all',month=c.month or 'all')">	<py:with vars="download_link=h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport',action='dataset_csv',id=c.publisher_name or 'all',month=c.month or 'all')">
<a class="btn button btn-primary btn-sm pull-right" href="${download_link}"><i class="icon-download"></i>  Download as CSV</a>	<a class="btn button btn-primary btn-sm pull-right" href="${download_link}"><i class="icon-download"></i>  Download as CSV</a>
</py:with>	</py:with>
<h1>Site Usage	<h1>Site Usage
<small py:if="c.publisher">${c.publisher.title}</small>	<small py:if="c.publisher">${c.publisher.title}</small>
<small py:if="not c.publisher">All datasets</small>	<small py:if="not c.publisher">All datasets</small>
</h1>	</h1>

<div class="row" style="background: #fff;">	<div class="row" style="background: #fff;">
<div class="col-md-8">	<div class="col-md-8">
<div class="whitebox">	<div class="whitebox">
<py:if test="c.graph_data">	<py:if test="c.graph_data">
${rickshaw_graph(c.graph_data,'dataset-downloads',debug=True)}	${rickshaw_graph(c.graph_data,'dataset-downloads',debug=True)}
</py:if>	</py:if>
</div>	</div>
</div>	</div>
</div>	</div>
<hr/>	<hr/>
<py:if test="c.month">	<py:if test="c.month">
<h4>Statistics for ${h.month_option_title(c.month,c.months,c.day)}:</h4>	<h4>Statistics for ${h.month_option_title(c.month,c.months,c.day)}:</h4>
</py:if>	</py:if>
<py:if test="not c.month">	<py:if test="not c.month">
<h2>Statistics for all months</h2>	<h2>Statistics for all months</h2>
</py:if>	</py:if>
<form style="margin-bottom:10px;" class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport',action='read')}" method="get">	<form style="margin-bottom:10px;" class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport',action='read')}" method="get">
<div class="controls">	<div class="controls">

${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}

<select name="publisher">	<select name="publisher">
<option value='' py:attrs="{'selected': 'selected' if not c.publisher else None}">All publishers</option>	<option value='' py:attrs="{'selected': 'selected' if not c.publisher else None}">All publishers</option>
<py:for each="val,desc in c.publishers">	<py:for each="val,desc in c.publishers">
<option value='${val}' py:attrs="{'selected': 'selected' if c.publisher_name == val else None}">${desc}</option>	<option value='${val}' py:attrs="{'selected': 'selected' if c.publisher_name == val else None}">${desc}</option>
</py:for>	</py:for>
</select>	</select>
<input class="btn button btn-primary btn-xs" type='submit' value="Update"/>	<input class="btn button btn-primary btn-xs" type='submit' value="Update"/>
</div>	</div>
</form>	</form>
<div class="alert alert-info" py:if="not c.top_packages">No page views in this period.</div>	<div class="alert alert-info" py:if="not c.top_packages">No page views in this period.</div>
<py:if test="c.top_packages">	<py:if test="c.top_packages">
<table class="ga-reports-table table table-condensed table-bordered">	<table class="ga-reports-table table table-condensed table-bordered">
<tr>	<tr>
<th>Dataset</th>	<th>Dataset</th>
<th>Views</th>	<th>Views</th>
<th>Downloads</th>	<th>Downloads</th>
</tr>	</tr>
<py:for each="package, views, visits,downloads in c.top_packages">	<py:for each="package, views, visits,downloads in c.top_packages">
<tr>	<tr>
<td>	<td>
${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}	${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}
</td>	</td>
<td class="td-numeric">${views}</td>	<td class="td-numeric">${views}</td>
<td class="td-numeric">${downloads}</td>	<td class="td-numeric">${downloads}</td>
</tr>	</tr>
</py:for>	</py:for>
</table>	</table>
</py:if>	</py:if>

${ga_footer()}	${ga_footer()}

</div>	</div>

<xi:include href="../site/layout.html" />	<xi:include href="../site/layout.html" />
</html>	</html>

file:a/ckanext/ga_report/templates/ga_report/site/index.html -> file:b/ckanext/ga_report/templates/ga_report/site/index.html

<html xmlns:py="http://genshi.edgewall.org/"	<html xmlns:py="http://genshi.edgewall.org/"
xmlns:i18n="http://genshi.edgewall.org/i18n"	xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:xi="http://www.w3.org/2001/XInclude"	xmlns:xi="http://www.w3.org/2001/XInclude"
py:strip="">	py:strip="">

<xi:include href="../ga_util.html" />	<xi:include href="../ga_util.html" />

<py:def function="page_title">Site usage</py:def>	<py:def function="page_title">Site usage</py:def>

<py:def function="optional_head">	<py:def function="optional_head">
</py:def>	</py:def>

<py:match path="breadcrumbs">	<py:match path="breadcrumbs">
<li><a href="/site-usage">Site Analytics</a></li>	<li><a href="/site-usage">Site Analytics</a></li>
<li><a href="/site-usage">Site-wide</a></li>	<li><a href="/site-usage">Site-wide</a></li>
</py:match>	</py:match>

<div py:match="content">	<div py:match="content">
<div class="row">	<div class="row">
<div class="col-sm-7 col-md-8 col-lg-9">	<div class="col-sm-7 col-md-8 col-lg-9">
<py:with vars="download_link=h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month or 'all')">	<py:with vars="download_link=h.url_for(controller='ckanext.ga_report.controller:GaReport',action='csv',month=c.month or 'all')">
<a class="btn button btn-primary btn-sm pull-right" href="${download_link}"><i class="icon-download"></i>  Download as CSV</a>	<a class="btn button btn-primary btn-sm pull-right" href="${download_link}"><i class="icon-download"></i>  Download as CSV</a>
</py:with>	</py:with>
<h1>Site Usage</h1>	<h1>Site Usage</h1>
</div>	</div>
<div class="col-sm-5 col-md-4 col-lg-3">	<div class="col-sm-5 col-md-4 col-lg-3">
<div class="panel panel-default">	<div class="panel panel-default">
<div class="panel-heading"><strong>Jump To...</strong></div>	<div class="panel-heading"><strong>Jump To...</strong></div>
<div class="panel-body">	<div class="panel-body">
<ul>	<ul>
<li><a href="/site-usage/publisher">Publisher Usage Statistics</a></li>	<li><a href="/site-usage/publisher">Publisher Usage Statistics</a></li>
<li><a href="/site-usageset">Dataset Usage Statistics</a></li>	<li><a href="/site-usage/dataset">Dataset Usage Statistics</a></li>
</ul>	</ul>
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>

<div class="row" style="background: #fff;">	<div class="row" style="background: #fff;">
<div class="col-md-8">	<div class="col-md-8">
<div class="whitebox">	<div class="whitebox">
<div class="tabbable">	<div class="tabbable">
<ul class="nav nav-tabs">	<ul class="nav nav-tabs">
<li class="active"><a href="#totals" data-hash="totals" data-toggle="tab">Totals</a></li>	<li class="active"><a href="#totals" data-hash="totals" data-toggle="tab">Totals</a></li>
<li class="dropdown">	<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Browsers	<a href="#" class="dropdown-toggle" data-toggle="dropdown">Browsers
<b class="caret"></b></a>	<b class="caret"></b></a>
<ul class="dropdown-menu">	<ul class="dropdown-menu">
<li><a href="#browsers_names" data-hash="browsers_names" data-toggle="tab">Browsers</a></li>	<li><a href="#browsers_names" data-hash="browsers_names" data-toggle="tab">Browsers</a></li>
<li><a href="#browsers_versions" data-hash="browsers_versions" data-toggle="tab">Versions</a></li>	<li><a href="#browsers_versions" data-hash="browsers_versions" data-toggle="tab">Versions</a></li>
</ul>	</ul>
</li>	</li>
<li class="dropdown">	<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Operating Systems	<a href="#" class="dropdown-toggle" data-toggle="dropdown">Operating Systems
<b class="caret"></b></a>	<b class="caret"></b></a>
<ul class="dropdown-menu">	<ul class="dropdown-menu">
<li><a href="#os" data-hash="os" data-toggle="tab">Operating Systems</a></li>	<li><a href="#os" data-hash="os" data-toggle="tab">Operating Systems</a></li>
<li><a href="#os_versions" data-hash="os_versions" data-toggle="tab">Versions</a></li>	<li><a href="#os_versions" data-hash="os_versions" data-toggle="tab">Versions</a></li>
</ul>	</ul>
</li>	</li>
<li class="dropdown">	<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Social	<a href="#" class="dropdown-toggle" data-toggle="dropdown">Social
<b class="caret"></b></a>	<b class="caret"></b></a>
<ul class="dropdown-menu">	<ul class="dropdown-menu">
<li><a href="#social_networks" data-hash="social_networks" data-toggle="tab">All networks</a></li>	<li><a href="#social_networks" data-hash="social_networks" data-toggle="tab">All networks</a></li>
<li><a href="#social_referrals_totals" data-hash="social_referrals_totals" data-toggle="tab">Referral links</a></li>	<li><a href="#social_referrals_totals" data-hash="social_referrals_totals" data-toggle="tab">Referral links</a></li>
</ul>	</ul>
</li>	</li>
<li><a href="#languages" data-hash="languages" data-toggle="tab">Languages</a></li>	<li><a href="#languages" data-hash="languages" data-toggle="tab">Languages</a></li>
<li><a href="#country" data-hash="country" data-toggle="tab">Country</a></li>	<li><a href="#country" data-hash="country" data-toggle="tab">Country</a></li>
</ul>	</ul>
<div class="tab-content">	<div class="tab-content">
<div class="tab-pane active" id="totals">	<div class="tab-pane active" id="totals">
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<table class="ga-reports-table table table-condensed table-bordered table-striped">	<table class="ga-reports-table table table-condensed table-bordered table-striped">
<tr>	<tr>
<th>Name</th>	<th>Name</th>
<th class="td-numeric">Value</th>	<th class="td-numeric">Value</th>
<th>History</th>	<th>History</th>
</tr>	</tr>
<py:for each="name, value, graph in c.global_totals">	<py:for each="name, value, graph in c.global_totals">
<tr>	<tr>
<td>${name}</td>	<td>${name}</td>
<td class="td-numeric">${value}</td>	<td class="td-numeric">${value}</td>
<td class="sparkline-cell">	<td class="sparkline-cell">
<span class="sparkline" sparkTooltips="${','.join([x for x,y in graph])}">	<span class="sparkline" sparkTooltips="${','.join([x for x,y in graph])}">
${','.join([y for x,y in graph])}	${','.join([y for x,y in graph])}
</span>	</span>
</td>	</td>
</tr>	</tr>
</py:for>	</py:for>
</table>	</table>
</div>	</div>
<div class="tab-pane" id="browsers_versions">	<div class="tab-pane" id="browsers_versions">
${rickshaw_graph(c.browser_versions_graph,'browser-versions',mode='stack')}	${rickshaw_graph(c.browser_versions_graph,'browser-versions',mode='stack')}
<hr/>	<hr/>
<p>Note: Where a browser has a large number of versions, these have been grouped together.</p>	<p>Note: Where a browser has a large number of versions, these have been grouped together.</p>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.browser_versions)}	${stat_table(c.browser_versions)}
</div>	</div>
<div class="tab-pane" id="browsers_names">	<div class="tab-pane" id="browsers_names">
${rickshaw_graph(c.browsers_graph,'browsers',mode='stack')}	${rickshaw_graph(c.browsers_graph,'browsers',mode='stack')}
<hr/>	<hr/>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.browsers)}	${stat_table(c.browsers)}
</div>	</div>
<div class="tab-pane" id="os">	<div class="tab-pane" id="os">
${rickshaw_graph(c.os_graph,'os',mode='stack')}	${rickshaw_graph(c.os_graph,'os',mode='stack')}
<hr/>	<hr/>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.os)}	${stat_table(c.os)}
</div>	</div>
<div class="tab-pane" id="os_versions">	<div class="tab-pane" id="os_versions">
${rickshaw_graph(c.os_versions_graph,'os_versions',mode='stack')}	${rickshaw_graph(c.os_versions_graph,'os_versions',mode='stack')}
<hr/>	<hr/>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.os_versions)}	${stat_table(c.os_versions)}
</div>	</div>
<div class="tab-pane" id="social_referrals_totals">	<div class="tab-pane" id="social_referrals_totals">
<p>Number of visits that were referred from social networks</p>	<p>Number of visits that were referred from social networks</p>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
${social_table(c.social_referrer_totals)}	${social_table(c.social_referrer_totals)}
</div>	</div>
<div class="tab-pane" id="social_networks">	<div class="tab-pane" id="social_networks">
${rickshaw_graph(c.social_networks_graph, 'social_networks',mode='stack')}	${rickshaw_graph(c.social_networks_graph, 'social_networks',mode='stack')}
<hr/>	<hr/>
<p>Percentage of visits that were referred from these social networks</p>	<p>Percentage of visits that were referred from these social networks</p>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.social_networks, 'Visits')}	${stat_table(c.social_networks, 'Visits')}
</div>	</div>
<div class="tab-pane" id="languages">	<div class="tab-pane" id="languages">
${rickshaw_graph(c.languages_graph,'languages',mode='stack')}	${rickshaw_graph(c.languages_graph,'languages',mode='stack')}
<hr/>	<hr/>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.languages)}	${stat_table(c.languages)}
</div>	</div>
<div class="tab-pane" id="country">	<div class="tab-pane" id="country">
${rickshaw_graph(c.country_graph,'country',mode='stack')}	${rickshaw_graph(c.country_graph,'country',mode='stack')}
<hr/>	<hr/>
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">	<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" method="get">
<h4 class="ga-reports-heading">Show stats table for:</h4>	<h4 class="ga-reports-heading">Show stats table for:</h4>
${month_selector(c.month, c.months, c.day)}	${month_selector(c.month, c.months, c.day)}
</form>	</form>
<hr/>	<hr/>
${stat_table(c.country)}	${stat_table(c.country)}
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>

</div>	</div>


<py:def function="optional_footer">	<py:def function="optional_footer">
<script type="text/javascript">	<script type="text/javascript">
$(function() {	$(function() {
CKAN.GA_Reports.bind_sparklines();	CKAN.GA_Reports.bind_sparklines();
CKAN.GA_Reports.bind_sidebar();	CKAN.GA_Reports.bind_sidebar();
CKAN.GA_Reports.bind_month_selector();	CKAN.GA_Reports.bind_month_selector();
});	});
</script>	</script>
</py:def>	</py:def>

<xi:include href="layout.html" />	<xi:include href="layout.html" />
</html>	</html>

file:a/ckanext/ga_report/templates/ga_report/site/layout_base.html -> file:b/ckanext/ga_report/templates/ga_report/site/layout_base.html

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"	<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<!--[if IE 8 ]> <html class="no-js ie8" lang="en"> <![endif]-->	<!--[if IE 8 ]> <html class="no-js ie8" lang="en"> <![endif]-->
<!--[if (gte IE 9)\|!(IE)]><!--> <html class="no-js" lang="en"	<!--[if (gte IE 9)\|!(IE)]><!--> <html class="no-js" lang="en"
xmlns="http://www.w3.org/1999/xhtml"	xmlns="http://www.w3.org/1999/xhtml"
xmlns:i18n="http://genshi.edgewall.org/i18n"	xmlns:i18n="http://genshi.edgewall.org/i18n"
xmlns:py="http://genshi.edgewall.org/"	xmlns:py="http://genshi.edgewall.org/"
xmlns:xi="http://www.w3.org/2001/XInclude"	xmlns:xi="http://www.w3.org/2001/XInclude"
> <!--<![endif]-->	> <!--<![endif]-->
<xi:include href="../../_util.html" />	<xi:include href="../../_util.html" />
<head>	<head>
<meta charset="utf-8" />	<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />	<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />

<title>${page_title()} - ${g.site_title}</title>	<title>${page_title()} - ${g.site_title}</title>
<meta name="description" content="" />	<meta name="description" content="" />
<meta name="author" content="" />	<meta name="author" content="" />

<meta name="viewport" content="width=device-width, initial-scale=1.0" />	<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="shortcut icon" href="${h.url_for_static(g.favicon)}" />	<link rel="shortcut icon" href="${h.url_for_static(g.favicon)}" />

<py:choose>	<py:choose>
<py:when test="defined('optional_feed')">	<py:when test="defined('optional_feed')">
${optional_feed()}	${optional_feed()}
</py:when>	</py:when>
<py:otherwise>	<py:otherwise>
<link rel="alternate" type="application/atom+xml" title="${g.site_title} - Recent Revision History" href="${h.url_for(controller='revision', action='list', format='atom', days=1)}" />	<link rel="alternate" type="application/atom+xml" title="${g.site_title} - Recent Revision History" href="${h.url_for(controller='revision', action='list', format='atom', days=1)}" />
</py:otherwise>	</py:otherwise>
</py:choose>	</py:choose>
<link href='http://fonts.googleapis.com/css?family=Ubuntu:400,700' rel='stylesheet' type='text/css' />	<link href='http://fonts.googleapis.com/css?family=Ubuntu:400,700' rel='stylesheet' type='text/css' />

<link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/jqueryui/1.8.14/css/jquery-ui.custom.css')}" type="text/css" media="screen, print" />	<link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/jqueryui/1.8.14/css/jquery-ui.custom.css')}" type="text/css" media="screen, print" />
<link rel="stylesheet" href="${h.url_for_static('/css/bootstrap.min.css')}" type="text/css" media="screen, projection" />	<link rel="stylesheet" href="${h.url_for_static('/css/bootstrap.min.css')}" type="text/css" media="screen, projection" />
<link rel="stylesheet" href="${h.url_for_static('/css/chosen.css')}" type="text/css" />	<link rel="stylesheet" href="${h.url_for_static('/css/chosen.css')}" type="text/css" />
<link rel="stylesheet" href="${h.url_for_static('/css/style.css?v=2')}" />	<link rel="stylesheet" href="${h.url_for_static('/css/style.css?v=2')}" />
${jsConditionalForIe(9, '<script type="text/javascript" src="' + h.url_for_static('/scripts/vendor/html5shiv/html5.js') + '"></script>')}	${jsConditionalForIe(9, '<script type="text/javascript" src="' + h.url_for_static('/scripts/vendor/html5shiv/html5.js') + '"></script>')}
<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>	<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>
<script src="//code.jquery.com/jquery-migrate-1.2.1.min.js"></script>	<script src="//code.jquery.com/jquery-migrate-1.2.1.min.js"></script>
<link rel="stylesheet" type="text/css" href="/scripts/vendor/rickshaw.min.css"/>	<link rel="stylesheet" type="text/css" href="/scripts/vendor/rickshaw.min.css"/>
<link rel="stylesheet" type="text/css" href="/css/ga_report.css?1"/>	<link rel="stylesheet" type="text/css" href="/css/ga_report.css?1"/>
<script type="text/javascript" src="/scripts/modernizr-2.6.2.custom.js"></script>	<script type="text/javascript" src="/scripts/modernizr-2.6.2.custom.js"></script>
<script type="text/javascript" src="/scripts/vendor/jquery.sparkline.modified.js"></script>	<script type="text/javascript" src="/scripts/vendor/jquery.sparkline.modified.js"></script>
<script type="text/javascript" src="/scripts/ckanext_ga_reports.js?1"></script>	<script type="text/javascript" src="/scripts/ckanext_ga_reports.js?1"></script>
<script type="text/javascript" src="/scripts/rickshaw_ie7_shim.js"></script>	<script type="text/javascript" src="/scripts/rickshaw_ie7_shim.js"></script>
<script type="text/javascript" src="/scripts/vendor/d3.v2.js"></script>	<script type="text/javascript" src="/scripts/vendor/d3.v2.js"></script>
<script type="text/javascript" src="/scripts/vendor/d3.layout.min.js"></script>	<script type="text/javascript" src="/scripts/vendor/d3.layout.min.js"></script>
<script type="text/javascript" src="/scripts/vendor/rickshaw.min.js"></script>	<script type="text/javascript" src="/scripts/vendor/rickshaw.min.js"></script>
<style>	<style>
#pre-content #breadcrumbs {	#pre-content #breadcrumbs {
padding-left: 0;	padding-left: 0;
padding-top: 10px;	padding-top: 10px;
padding-bottom: 10px;	padding-bottom: 10px;
margin-bottom: 0;	margin-bottom: 0;
list-style-type: none;	list-style-type: none;
color: #ccc	color: #ccc
}	}

#pre-content #breadcrumbs li {	#pre-content #breadcrumbs li {
color: #999;	color: #999;
display: inline-block;	display: inline-block;
zoom: 1;	zoom: 1;
*display: inline;	*display: inline;
margin-right: 8px	margin-right: 8px
}	}

#pre-content #breadcrumbs li:after {	#pre-content #breadcrumbs li:after {
content: '/';	content: '/';
display: inline-block;	display: inline-block;
zoom: 1;	zoom: 1;
*display: inline;	*display: inline;
margin-left: 12px	margin-left: 12px
}	}

#pre-content #breadcrumbs li:last-child:after {	#pre-content #breadcrumbs li:last-child:after {
display: none	display: none
}	}

#pre-content #breadcrumbs a {	#pre-content #breadcrumbs a {
color: #999	color: #999
}	}

#pre-content #breadcrumbs a:hover {	#pre-content #breadcrumbs a:hover {
color: #000;	color: #000;
text-decoration: none	text-decoration: none
}	}

#pre-content #breadcrumbs .spacer {	#pre-content #breadcrumbs .spacer {
color: #ccc	color: #ccc
}	}
</style>	</style>
<py:if test="defined('optional_head')">	<py:if test="defined('optional_head')">
${optional_head()}	${optional_head()}
</py:if>	</py:if>

${h.literal(getattr(g, 'template_head_end', ''))}	${h.literal(getattr(g, 'template_head_end', ''))}
</head>	</head>

<body class="${request.environ.get('pylons.routes_dict', {}).get('action')}	<body class="${request.environ.get('pylons.routes_dict', {}).get('action')}
${request.environ.get('pylons.routes_dict', {}).get('controller').split(':')[-1]}	${request.environ.get('pylons.routes_dict', {}).get('controller').split(':')[-1]}
${defined('body_class') and body_class()}	${defined('body_class') and body_class()}
">	">

<div id="wrap">	<div id="wrap">
<div class="header outer">	<div class="header outer">
<header class="container">	<header class="container">
<div class="menu account">	<div class="menu account">
<span class="ckan-logged-in" style="display: none;">	<span class="ckan-logged-in" style="display: none;">
<a href="${h.url_for(controller='user',action='me')}">${h.gravatar((c.userobj.email_hash if c and c.userobj else ''),size=20)}${c.user}</a>	<a href="${h.url_for(controller='user',action='me')}">${h.gravatar((c.userobj.email_hash if c and c.userobj else ''),size=20)}${c.user}</a>
<a href="${h.url_for('/user/_logout')}">Logout</a>	<a href="${h.url_for('/user/_logout')}">Logout</a>
</span>	</span>
<span class="ckan-logged-out">	<span class="ckan-logged-out">
<a href="${h.url_for(controller='user',action='login')}">Login</a>	<a href="${h.url_for(controller='user',action='login')}">Login</a>
<a href="${h.url_for(controller='user',action='register')}">Register</a>	<a href="${h.url_for(controller='user',action='register')}">Register</a>
</span>	</span>
</div>	</div>
<a href="${h.url('home')}">	<a href="${h.url('home')}">
<img width="64" src="${h.url_for_static(g.site_logo)}" alt="${g.site_title} Logo" title="${g.site_title} Logo" id="logo" />	<img width="196" src="${h.url_for_static(g.site_logo)}" alt="${g.site_title} Logo" title="${g.site_title} Logo" id="logo" />
</a>	</a>
<div id="site-name">	<div id="site-name">
<!-- <a href="${h.url('home')}">${g.site_title} — ${g.site_description}</a>-->	<!-- <a href="${h.url('home')}">${g.site_title} — ${g.site_description}</a>-->
</div>	</div>
<div class="menu">	<div class="menu">
<!-- <span id="menusearch">	<!-- <span id="menusearch">
<form action="${h.url(controller='package', action='search')}" method="GET">	<form action="${h.url(controller='package', action='search')}" method="GET">
<input name="q" value="${c.q if hasattr(c, 'q') else ''}" class="search" placeholder="${_('Find datasets')}" />	<input name="q" value="${c.q if hasattr(c, 'q') else ''}" class="search" placeholder="${_('Find datasets')}" />
</form>	</form>
</span>	</span>
<div id="mainmenu">	<div id="mainmenu">
<span py:if="h.check_access('package_create')">${h.nav_link(_('Add a dataset'), controller='package', action='new')}</span>	<span py:if="h.check_access('package_create')">${h.nav_link(_('Add a dataset'), controller='package', action='new')}</span>
${h.nav_link(_('Search'), controller='package', action='search', highlight_actions = 'new index')}	${h.nav_link(_('Search'), controller='package', action='search', highlight_actions = 'new index')}
${h.nav_link(_('Groups'), named_route='%s_index' % h.default_group_type())}	${h.nav_link(_('Groups'), named_route='%s_index' % h.default_group_type())}
${h.nav_link(_('About'), controller='home', action='about')}	${h.nav_link(_('About'), controller='home', action='about')}
</div>-->	</div>-->
</div>	</div>
</header>	</header>
</div>	</div>
<div id="pre-content">	<div id="pre-content">
<div class="container">	<div class="container">
<div class="row">	<div class="row">
<div class="col-md-12">	<div class="col-md-12">
<ul id="breadcrumbs">	<ul id="breadcrumbs">
<li><a href="/"><i class="icon-home"></i></a></li>	<li><a href="/"><i class="icon-home"></i></a></li>
<breadcrumbs>	<breadcrumbs>
</breadcrumbs>	</breadcrumbs>
</ul>	</ul>
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>

<py:with vars="messages = list(h.flash.pop_messages())">	<py:with vars="messages = list(h.flash.pop_messages())">
<div class="flash-messages container">	<div class="flash-messages container">
<div class="alert ${m.category}" py:for="m in messages">	<div class="alert ${m.category}" py:for="m in messages">
${h.literal(m)}	${h.literal(m)}
</div>	</div>
</div>	</div>
</py:with>	</py:with>

<div id="main" class="container" role="main">	<div id="main" class="container" role="main">
<h1 py:if="defined('page_heading')" class="page_heading">	<h1 py:if="defined('page_heading')" class="page_heading">
<img py:if="defined('page_logo')" id="page-logo" src="${page_logo()}" alt="Page Logo" />	<img py:if="defined('page_logo')" id="page-logo" src="${page_logo()}" alt="Page Logo" />
<!-- ${page_heading()} -->	<!-- ${page_heading()} -->
</h1>	</h1>
<div class="row">	<div class="row">
<div class="span12">	<div class="span12">
<div id="minornavigation">	<div id="minornavigation">
<minornavigation></minornavigation>	<minornavigation></minornavigation>
</div>	</div>
</div>	</div>
</div>	</div>

<div class="row">	<div class="row">
<div class="span9 content-outer">	<div class="span9 content-outer">
<div id="content">	<div id="content">
<py:if test="defined('content')">	<py:if test="defined('content')">
${content()}	${content()}
</py:if>	</py:if>
<content>	<content>
<p>Master content template placeholder … please replace me.</p>	<p>Master content template placeholder … please replace me.</p>
</content>	</content>
</div> <!-- /content -->	</div> <!-- /content -->
</div>	</div>
<div class="span3 sidebar-outer">	<div class="span3 sidebar-outer">
<div id="sidebar">	<div id="sidebar">
<div class="col-md-4">	<div class="col-md-4">
<div class="whitebox">	<div class="whitebox">
<strong>Graph Legend</strong>	<strong>Graph Legend</strong>
<div id="graph-legend-container">	<div id="graph-legend-container">
<div style="display: none;" id="legend_none">(No graph is loaded)</div>	<div style="display: none;" id="legend_none">(No graph is loaded)</div>
</div>	</div>
</div>	</div>
</div>	</div>

<ul class="widget-list">	<ul class="widget-list">
<py:if test="defined('primary_sidebar_extras')">	<py:if test="defined('primary_sidebar_extras')">
${primary_sidebar_extras()}	${primary_sidebar_extras()}
</py:if>	</py:if>
<primarysidebar>	<primarysidebar>
<!-- Primary Side Bar Goes Here -->	<!-- Primary Side Bar Goes Here -->
</primarysidebar>	</primarysidebar>
</ul>	</ul>
</div>	</div>
</div>	</div>
</div>	</div>
</div>	</div>
<br/><br/>	<br/><br/>
</div>	</div>
<div class="clearfix"></div>	<div class="clearfix"></div>
<div class="footer outer">	<div class="footer outer">
<footer class="container">	<footer class="container">
<div class="row">	<div class="row">
<!-- <div class="span3">	<!-- <div class="span3">
<h3 class="widget-title">About ${g.site_title}</h3>	<h3 class="widget-title">About ${g.site_title}</h3>
<div class="textwidget">	<div class="textwidget">
<ul>	<ul>
<li>${h.link_to(_('About'), h.url_for(controller='home', action='about'))}</li>	<li>${h.link_to(_('About'), h.url_for(controller='home', action='about'))}</li>
<li>	<li>
<a href="http://twitter.com/ckanproject">Twitter @ckanproject</a>	<a href="http://twitter.com/ckanproject">Twitter @ckanproject</a>
</li>	</li>
<li>${h.link_to(_('API'), h.url_for(controller='api', action='get_api', ver=1))}</li>	<li>${h.link_to(_('API'), h.url_for(controller='api', action='get_api', ver=1))}</li>
<li>${h.link_to(_('API Docs'), 'http://docs.ckan.org/en/latest/api.html')}</li>	<li>${h.link_to(_('API Docs'), 'http://docs.ckan.org/en/latest/api.html')}</li>
<li>	<li>
<a href="http://ckan.org/contact/">Contact Us</a>	<a href="http://ckan.org/contact/">Contact Us</a>
</li>	</li>
<li>	<li>
<a href="http://okfn.org/privacy-policy/">Privacy Policy</a>	<a href="http://okfn.org/privacy-policy/">Privacy Policy</a>
</li>	</li>
</ul>	</ul>
</div>	</div>
</div>	</div>
<div class="span3">	<div class="span3">
<h3 class="widget-title">Sections</h3>	<h3 class="widget-title">Sections</h3>
<div class="textwidget">	<div class="textwidget">
<ul>	<ul>
<li>	<li>
<a href="${h.url(controller='user', action='index')}">	<a href="${h.url(controller='user', action='index')}">
Users	Users
</a>	</a>
</li>	</li>
<li>	<li>
<a href="${h.url(controller='tag', action='index')}">	<a href="${h.url(controller='tag', action='index')}">
Tags	Tags
</a>	</a>
</li>	</li>
<li py:if="'stats' in config.get('ckan.plugins','').split(' ')">	<li py:if="'stats' in config.get('ckan.plugins','').split(' ')">
<a href="${h.url('stats')}">	<a href="${h.url('stats')}">
Statistics	Statistics
</a>	</a>
</li>	</li>
<li>	<li>
<a href="${h.url(controller='revision', action='index')}">	<a href="${h.url(controller='revision', action='index')}">
Revisions	Revisions
</a>	</a>
</li>	</li>
<li>	<li>
<a href="${h.url_for('ckanadmin_index')}">	<a href="${h.url_for('ckanadmin_index')}">
Site Admin	Site Admin
</a>	</a>
</li>	</li>
</ul>	</ul>
</div>	</div>
</div>	</div>
<div class="span3">	<div class="span3">
<h3 class="widget-title">Languages</h3>	<h3 class="widget-title">Languages</h3>
<div class="textwidget">	<div class="textwidget">
<ul>	<ul>
<?python	<?python
current_url = request.environ['CKAN_CURRENT_URL']	current_url = request.environ['CKAN_CURRENT_URL']
?>	?>
<li py:for="locale in h.get_available_locales()">	<li py:for="locale in h.get_available_locales()">
<a href="${h.url(current_url, locale=str(locale))}">	<a href="${h.url(current_url, locale=str(locale))}">
${locale.display_name or locale.english_name}	${locale.display_name or locale.english_name}
</a>	</a>
</li>	</li>
</ul>	</ul>
</div>	</div>
</div>-->	</div>-->
<div class="span12">	<div class="span12">
<p id="credits">	<p id="credits">
© 2012-2014	© 2012-2014
<img src="//assets.okfn.org/images/logo/okf_logo_white_and_green_tiny.png" id="footer-okf-logo" />	<img src="//assets.okfn.org/images/logo/okf_logo_white_and_green_tiny.png" id="footer-okf-logo" />
<a href="http://okfn.org/">Open Knowledge Foundation</a>	<a href="http://okfn.org/">Open Knowledge Foundation</a>
Licensed under the <a href="http://opendatacommons.org/licenses/odbl/1.0/">Open Database License</a>	Licensed under the <a href="http://opendatacommons.org/licenses/odbl/1.0/">Open Database License</a>
<a href="http://opendefinition.org/"><img alt="This Content and Data is Open" src="//assets.okfn.org/images/ok_buttons/od_80x15_blue.png" style="border: none;"/></a>	<a href="http://opendefinition.org/"><img alt="This Content and Data is Open" src="//assets.okfn.org/images/ok_buttons/od_80x15_blue.png" style="border: none;"/></a>

<br/><br/>	<br/><br/>
Powered by <a href="http://ckan.org">CKAN</a> v${c.__version__}.<br/>	Powered by <a href="http://ckan.org">CKAN</a> v${c.__version__}.<br/>
</p>	</p>
</div>	</div>
</div>	</div>
</footer>	</footer>
</div> <!-- eo #container -->	</div> <!-- eo #container -->

<div style="display:none;" id="scripts">	<div style="display:none;" id="scripts">
<!--<script src="${h.url_for_static('/scripts/vendor/jquery/1.7.1/jquery.js')}"></script>-->	<!--<script src="${h.url_for_static('/scripts/vendor/jquery/1.7.1/jquery.js')}"></script>-->
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/json2.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/json2.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.tmpl/beta1/jquery.tmpl.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.tmpl/beta1/jquery.tmpl.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.cookie/jquery.cookie.min.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.cookie/jquery.cookie.min.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.chosen/0.9.7/chosen.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.chosen/0.9.7/chosen.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.placeholder/jquery.placeholder.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.placeholder/jquery.placeholder.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jqueryui/1.8.14/jquery-ui.min.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jqueryui/1.8.14/jquery-ui.min.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/bootstrap/2.0.3/bootstrap.min.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/bootstrap/2.0.3/bootstrap.min.js')}"></script>

<!-- for application.js -->	<!-- for application.js -->
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/underscore/1.1.6/underscore.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/underscore/1.1.6/underscore.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/backbone/0.5.1/backbone.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/backbone/0.5.1/backbone.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.fileupload/20110801/jquery.iframe-transport.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.fileupload/20110801/jquery.iframe-transport.js')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.fileupload/20110801/jquery.fileupload.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/vendor/jquery.fileupload/20110801/jquery.fileupload.js')}"></script>

<!-- Translated js strings live inside an html template. -->	<!-- Translated js strings live inside an html template. -->
<xi:include href="../../js_strings.html" />	<xi:include href="../../js_strings.html" />
<!-- finally our application js that sets everything up-->	<!-- finally our application js that sets everything up-->
<script type="text/javascript" src="${h.url_for_static('/scripts/application.js?lang=${c.locale}')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/application.js?lang=${c.locale}')}"></script>
<script type="text/javascript" src="${h.url_for_static('/scripts/templates.js')}"></script>	<script type="text/javascript" src="${h.url_for_static('/scripts/templates.js')}"></script>

<script type="text/javascript">	<script type="text/javascript">
CKAN.plugins = [	CKAN.plugins = [
// Declare js array from Python string	// Declare js array from Python string
${['\'%s\', '%s for s in config.get('ckan.plugins','').split(' ')]}	${['\'%s\', '%s for s in config.get('ckan.plugins','').split(' ')]}
];	];
<py:if test="config.get('ckan.storage.bucket', '')">	<py:if test="config.get('ckan.storage.bucket', '')">
CKAN.plugins.push('storage');	CKAN.plugins.push('storage');
</py:if>	</py:if>
CKAN.SITE_URL = '${h.url('/')}';	CKAN.SITE_URL = '${h.url('/')}';
CKAN.SITE_URL_NO_LOCALE = '${h.url('/', locale='default')}';	CKAN.SITE_URL_NO_LOCALE = '${h.url('/', locale='default')}';
CKAN.LANG = '${h.lang()}';	CKAN.LANG = '${h.lang()}';
// later use will add offsets with leading '/' so ensure no trailing slash	// later use will add offsets with leading '/' so ensure no trailing slash
CKAN.SITE_URL = CKAN.SITE_URL.replace(/\/$/, '');	CKAN.SITE_URL = CKAN.SITE_URL.replace(/\/$/, '');
CKAN.SITE_URL_NO_LOCALE = CKAN.SITE_URL_NO_LOCALE.replace(/\/$/, '');	CKAN.SITE_URL_NO_LOCALE = CKAN.SITE_URL_NO_LOCALE.replace(/\/$/, '');
$(document).ready(function() {	$(document).ready(function() {
var ckan_user = '${c.user}';	var ckan_user = '${c.user}';
if (ckan_user) {	if (ckan_user) {
$(".ckan-logged-out").hide();	$(".ckan-logged-out").hide();
$(".ckan-logged-in").show();	$(".ckan-logged-in").show();
}	}
$('input[placeholder], textarea[placeholder]').placeholder();	$('input[placeholder], textarea[placeholder]').placeholder();

$(".chzn-select").chosen();	$(".chzn-select").chosen();
});	});
</script>	</script>

<py:if test="h.asbool(config.get('ckan.tracking_enabled', 'false'))">	<py:if test="h.asbool(config.get('ckan.tracking_enabled', 'false'))">
${ h.snippet('snippets/internal-tracking.html') }	${ h.snippet('snippets/internal-tracking.html') }
</py:if>	</py:if>
</div> <!-- #scripts -->	</div> <!-- #scripts -->

<py:if test="defined('optional_footer')">	<py:if test="defined('optional_footer')">
${optional_footer()}	${optional_footer()}
</py:if>	</py:if>

${h.literal(getattr(g, 'template_footer_end', ''))}	${h.literal(getattr(g, 'template_footer_end', ''))}
</body>	</body>
</html>	</html>