import re | import re |
import csv | import csv |
import sys | import sys |
import logging | import logging |
import operator | import operator |
import collections | import collections |
from ckan.lib.base import (BaseController, c, g, render, request, response, abort) | from ckan.lib.base import (BaseController, c, g, render, request, response, abort) |
import sqlalchemy | import sqlalchemy |
from sqlalchemy import func, cast, Integer | from sqlalchemy import func, cast, Integer |
import ckan.model as model | import ckan.model as model |
from ga_model import GA_Url, GA_Stat, GA_ReferralStat | from ga_model import GA_Url, GA_Stat, GA_ReferralStat |
log = logging.getLogger('ckanext.ga-report') | log = logging.getLogger('ckanext.ga-report') |
def _get_month_name(strdate): | def _get_month_name(strdate): |
import calendar | import calendar |
from time import strptime | from time import strptime |
d = strptime(strdate, '%Y-%m') | d = strptime(strdate, '%Y-%m') |
return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) | return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) |
def _month_details(cls): | def _month_details(cls): |
months = [] | months = [] |
vals = model.Session.query(cls.period_name).distinct().all() | vals = model.Session.query(cls.period_name).distinct().all() |
for m in vals: | for m in vals: |
months.append( (m[0], _get_month_name(m[0]))) | months.append( (m[0], _get_month_name(m[0]))) |
return sorted(months, key=operator.itemgetter(0), reverse=True) | return sorted(months, key=operator.itemgetter(0), reverse=True) |
class GaReport(BaseController): | class GaReport(BaseController): |
def csv(self, month): | def csv(self, month): |
import csv | import csv |
q = model.Session.query(GA_Stat) | q = model.Session.query(GA_Stat) |
if month != 'all': | if month != 'all': |
q = q.filter(GA_Stat.period_name==month) | q = q.filter(GA_Stat.period_name==month) |
entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() | entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() |
response.headers['Content-Type'] = "text/csv; charset=utf-8" | response.headers['Content-Type'] = "text/csv; charset=utf-8" |
response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) | response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) |
writer = csv.writer(response) | writer = csv.writer(response) |
writer.writerow(["Period", "Statistic", "Key", "Value"]) | writer.writerow(["Period", "Statistic", "Key", "Value"]) |
for entry in entries: | for entry in entries: |
writer.writerow([entry.period_name.encode('utf-8'), | writer.writerow([entry.period_name.encode('utf-8'), |
entry.stat_name.encode('utf-8'), | entry.stat_name.encode('utf-8'), |
entry.key.encode('utf-8'), | entry.key.encode('utf-8'), |
entry.value.encode('utf-8')]) | entry.value.encode('utf-8')]) |
def index(self): | def index(self): |
# Get the month details by fetching distinct values and determining the | # Get the month details by fetching distinct values and determining the |
# month names from the values. | # month names from the values. |
c.months = _month_details(GA_Stat) | c.months = _month_details(GA_Stat) |
# Work out which month to show, based on query params of the first item | # Work out which month to show, based on query params of the first item |
c.month_desc = 'all months' | c.month_desc = 'all months' |
c.month = request.params.get('month', '') | c.month = request.params.get('month', '') |
if c.month: | if c.month: |
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) | c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) |
q = model.Session.query(GA_Stat).\ | q = model.Session.query(GA_Stat).\ |
filter(GA_Stat.stat_name=='Totals') | filter(GA_Stat.stat_name=='Totals') |
if c.month: | if c.month: |
q = q.filter(GA_Stat.period_name==c.month) | q = q.filter(GA_Stat.period_name==c.month) |
entries = q.order_by('ga_stat.key').all() | entries = q.order_by('ga_stat.key').all() |
def clean_key(key, val): | def clean_key(key, val): |
if key in ['Average time on site', 'Pages per visit', 'New visits']: | if key in ['Average time on site', 'Pages per visit', 'New visits']: |
val = "%.2f" % round(float(val), 2) | val = "%.2f" % round(float(val), 2) |
if key == 'Average time on site': | if key == 'Average time on site': |
mins, secs = divmod(float(val), 60) | mins, secs = divmod(float(val), 60) |
hours, mins = divmod(mins, 60) | hours, mins = divmod(mins, 60) |
val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) | val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) |
if key == 'New visits': | if key == 'New visits': |
val = "%s%%" % val | val = "%s%%" % val |
if key in ['Bounces', 'Total page views', 'Total visits']: | if key in ['Bounces', 'Total page views', 'Total visits']: |
val = int(val) | val = int(val) |
return key, val | return key, val |
c.global_totals = [] | c.global_totals = [] |
if c.month: | if c.month: |
for e in entries: | for e in entries: |
key, val = clean_key(e.key, e.value) | key, val = clean_key(e.key, e.value) |
c.global_totals.append((key, val)) | c.global_totals.append((key, val)) |
else: | else: |
d = collections.defaultdict(list) | d = collections.defaultdict(list) |
for e in entries: | for e in entries: |
d[e.key].append(float(e.value)) | d[e.key].append(float(e.value)) |
for k, v in d.iteritems(): | for k, v in d.iteritems(): |
if k in ['Bounces', 'Total page views', 'Total visits']: | if k in ['Bounces', 'Total page views', 'Total visits']: |
v = sum(v) | v = sum(v) |
else: | else: |
v = float(sum(v))/len(v) | v = float(sum(v))/len(v) |
key, val = clean_key(k,v) | key, val = clean_key(k,v) |
c.global_totals.append((key, val)) | c.global_totals.append((key, val)) |
c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) | c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0)) |
keys = { | keys = { |
'Browser versions': 'browser_versions', | 'Browser versions': 'browser_versions', |
'Browsers': 'browsers', | 'Browsers': 'browsers', |
'Operating Systems versions': 'os_versions', | 'Operating Systems versions': 'os_versions', |
'Operating Systems': 'os', | 'Operating Systems': 'os', |
'Social sources': 'social_networks', | 'Social sources': 'social_networks', |
'Languages': 'languages', | 'Languages': 'languages', |
'Country': 'country' | 'Country': 'country' |
} | } |
def shorten_name(name, length=60): | def shorten_name(name, length=60): |
return (name[:length] + '..') if len(name) > 60 else name | return (name[:length] + '..') if len(name) > 60 else name |
def fill_out_url(url): | def fill_out_url(url): |
import urlparse | import urlparse |
return urlparse.urljoin(g.site_url, url) | return urlparse.urljoin(g.site_url, url) |
c.social_referrer_totals, c.social_referrers = [], [] | c.social_referrer_totals, c.social_referrers = [], [] |
q = model.Session.query(GA_ReferralStat) | q = model.Session.query(GA_ReferralStat) |
q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q | q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q |
q = q.order_by('ga_referrer.count::int desc') | q = q.order_by('ga_referrer.count::int desc') |
for entry in q.all(): | for entry in q.all(): |
c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), | c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), |
entry.source,entry.count)) | entry.source,entry.count)) |
q = model.Session.query(GA_ReferralStat.url, | q = model.Session.query(GA_ReferralStat.url, |
func.sum(GA_ReferralStat.count).label('count')) | func.sum(GA_ReferralStat.count).label('count')) |
q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q | q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q |
q = q.order_by('count desc').group_by(GA_ReferralStat.url) | q = q.order_by('count desc').group_by(GA_ReferralStat.url) |
for entry in q.all(): | for entry in q.all(): |
c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', | c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', |
entry[1])) | entry[1])) |
browser_version_re = re.compile("(.*)\((.*)\)") | browser_version_re = re.compile("(.*)\((.*)\)") |
for k, v in keys.iteritems(): | for k, v in keys.iteritems(): |
def clean_field(key): | def clean_field(key): |
if k != 'Browser versions': | if k != 'Browser versions': |
return key | return key |
m = browser_version_re.match(key) | m = browser_version_re.match(key) |
browser = m.groups()[0].strip() | browser = m.groups()[0].strip() |
ver = m.groups()[1] | ver = m.groups()[1] |
parts = ver.split('.') | parts = ver.split('.') |
if len(parts) > 1: | if len(parts) > 1: |
if parts[1][0] == '0': | if parts[1][0] == '0': |
ver = parts[0] | ver = parts[0] |
else: | else: |
ver = "%s.%s" % (parts[0],parts[1]) | ver = "%s.%s" % (parts[0],parts[1]) |
if browser in ['Safari','Android Browser']: # Special case complex version nums | if browser in ['Safari','Android Browser']: # Special case complex version nums |
ver = parts[0] | ver = parts[0] |
if len(ver) > 2: | if len(ver) > 2: |
ver = "%s%sX" % (ver[0], ver[1]) | ver = "%s%sX" % (ver[0], ver[1]) |
return "%s (%s)" % (browser, ver,) | return "%s (%s)" % (browser, ver,) |
q = model.Session.query(GA_Stat).\ | q = model.Session.query(GA_Stat).\ |
filter(GA_Stat.stat_name==k) | filter(GA_Stat.stat_name==k) |
if c.month: | if c.month: |
entries = [] | entries = [] |
q = q.filter(GA_Stat.period_name==c.month).\ | q = q.filter(GA_Stat.period_name==c.month).\ |
order_by('ga_stat.value::int desc') | order_by('ga_stat.value::int desc') |
d = collections.defaultdict(int) | d = collections.defaultdict(int) |
for e in q.all(): | for e in q.all(): |
d[clean_field(e.key)] += int(e.value) | d[clean_field(e.key)] += int(e.value) |
entries = [] | entries = [] |
for key, val in d.iteritems(): | for key, val in d.iteritems(): |
entries.append((key,val,)) | entries.append((key,val,)) |
entries = sorted(entries, key=operator.itemgetter(1), reverse=True) | entries = sorted(entries, key=operator.itemgetter(1), reverse=True) |
def percent(num, total): | def percent(num, total): |
p = 100 * float(num)/float(total) | p = 100 * float(num)/float(total) |
return "%.2f%%" % round(p, 2) | return "%.2f%%" % round(p, 2) |
# Get the total for each set of values and then set the value as | # Get the total for each set of values and then set the value as |
# a percentage of the total | # a percentage of the total |
if k == 'Social sources': | if k == 'Social sources': |
total = sum([x for n,x in c.global_totals if n == 'Total visits']) | total = sum([x for n,x in c.global_totals if n == 'Total visits']) |
else: | else: |
total = sum([num for _,num in entries]) | total = sum([num for _,num in entries]) |
setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) | setattr(c, v, [(k,percent(v,total)) for k,v in entries ]) |
return render('ga_report/site/index.html') | return render('ga_report/site/index.html') |
class GaPublisherReport(BaseController): | class GaPublisherReport(BaseController): |
""" | """ |
Displays the pageview and visit count for specific publishers based on | Displays the pageview and visit count for specific publishers based on |
the datasets associated with the publisher. | the datasets associated with the publisher. |
""" | """ |
def csv(self, month): | def csv(self, month): |
c.month = month if not month =='all' else '' | c.month = month if not month =='all' else '' |
response.headers['Content-Type'] = "text/csv; charset=utf-8" | response.headers['Content-Type'] = "text/csv; charset=utf-8" |
response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) | response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) |
writer = csv.writer(response) | writer = csv.writer(response) |
writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) | writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) |
for publisher,view,visit in self._get_publishers(None): | for publisher,view,visit in _get_publishers(None): |
writer.writerow([publisher.title.encode('utf-8'), | writer.writerow([publisher.title.encode('utf-8'), |
view, | view, |
visit, | visit, |
month]) | month]) |
def publisher_csv(self, id, month): | def publisher_csv(self, id, month): |
c.month = month if not month =='all' else '' | c.month = month if not month =='all' else '' |
c.publisher = model.Group.get(id) | c.publisher = model.Group.get(id) |
if not c.publisher: | if not c.publisher: |
abort(404, 'A publisher with that name could not be found') | abort(404, 'A publisher with that name could not be found') |
packages = self._get_packages(c.publisher) | packages = self._get_packages(c.publisher) |
response.headers['Content-Type'] = "text/csv; charset=utf-8" | response.headers['Content-Type'] = "text/csv; charset=utf-8" |
response.headers['Content-Disposition'] = \ | response.headers['Content-Disposition'] = \ |
str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,)) | str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,)) |
writer = csv.writer(response) | writer = csv.writer(response) |
writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) | writer.writerow(["Publisher", "Views", "Visits", "Period Name"]) |
for package,view,visit in packages: | for package,view,visit in packages: |
writer.writerow([package.title.encode('utf-8'), | writer.writerow([package.title.encode('utf-8'), |
view, | view, |
visit, | visit, |
month]) | month]) |
def index(self): | def index(self): |
# Get the month details by fetching distinct values and determining the | # Get the month details by fetching distinct values and determining the |
# month names from the values. | # month names from the values. |
c.months = _month_details(GA_Url) | c.months = _month_details(GA_Url) |
# Work out which month to show, based on query params of the first item | # Work out which month to show, based on query params of the first item |
c.month = request.params.get('month', '') | c.month = request.params.get('month', '') |
c.month_desc = 'all months' | c.month_desc = 'all months' |
if c.month: | if c.month: |
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) | c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) |
c.top_publishers = self._get_publishers() | c.top_publishers = _get_publishers() |
return render('ga_report/publisher/index.html') | return render('ga_report/publisher/index.html') |
def _get_publishers(self, limit=20): | |
connection = model.Session.connection() | |
q = """ | |
select department_id, sum(pageviews::int) views, sum(visitors::int) visits | |
from ga_url | |
where department_id <> ''""" | |
if c.month: | |
q = q + """ | |
and period_name=%s | |
""" | |
q = q + """ | |
group by department_id order by views desc | |
""" | |
if limit: | |
q = q + " limit %s;" % (limit) | |
# Add this back (before and period_name =%s) if you want to ignore publisher | |
# homepage views | |
# and not url like '/publisher/%%' | |
top_publishers = [] | |
res = connection.execute(q, c.month) | |
for row in res: | |
g = model.Group.get(row[0]) | |
if g: | |
top_publishers.append((g, row[1], row[2])) | |
return top_publishers | |
def _get_packages(self, publisher, count=-1): | def _get_packages(self, publisher, count=-1): |
if count == -1: | if count == -1: |
count = sys.maxint | count = sys.maxint |
top_packages = [] | top_packages = [] |
q = model.Session.query(GA_Url).\ | q = model.Session.query(GA_Url).\ |
filter(GA_Url.department_id==publisher.name).\ | filter(GA_Url.department_id==publisher.name).\ |
filter(GA_Url.url.like('/dataset/%')) | filter(GA_Url.url.like('/dataset/%')) |
if c.month: | if c.month: |
q = q.filter(GA_Url.period_name==c.month) | q = q.filter(GA_Url.period_name==c.month) |
q = q.order_by('ga_url.pageviews::int desc') | q = q.order_by('ga_url.pageviews::int desc') |
if c.month: | if c.month: |
for entry in q[:count]: | for entry in q[:count]: |
p = model.Package.get(entry.url[len('/dataset/'):]) | p = model.Package.get(entry.url[len('/dataset/'):]) |
top_packages.append((p,entry.pageviews,entry.visitors)) | top_packages.append((p,entry.pageviews,entry.visitors)) |
else: | else: |
ds = {} | ds = {} |
for entry in q.all(): | for entry in q.all(): |
if len(ds) >= count: | if len(ds) >= count: |
break | break |
p = model.Package.get(entry.url[len('/dataset/'):]) | p = model.Package.get(entry.url[len('/dataset/'):]) |
if not p in ds: | if not p in ds: |
ds[p] = {'views':0, 'visits': 0} | ds[p] = {'views':0, 'visits': 0} |
ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) | ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) |
ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) | ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) |
results = [] | results = [] |
for k, v in ds.iteritems(): | for k, v in ds.iteritems(): |
results.append((k,v['views'],v['visits'])) | results.append((k,v['views'],v['visits'])) |
top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) | top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) |
return top_packages | return top_packages |
def read(self, id): | def read(self, id): |
count = 20 | count = 20 |
c.publisher = model.Group.get(id) | c.publisher = model.Group.get(id) |
if not c.publisher: | if not c.publisher: |
abort(404, 'A publisher with that name could not be found') | abort(404, 'A publisher with that name could not be found') |
c.top_packages = [] # package, dataset_views in c.top_packages | c.top_packages = [] # package, dataset_views in c.top_packages |
# Get the month details by fetching distinct values and determining the | # Get the month details by fetching distinct values and determining the |
# month names from the values. | # month names from the values. |
c.months = _month_details(GA_Url) | c.months = _month_details(GA_Url) |
# Work out which month to show, based on query params of the first item | # Work out which month to show, based on query params of the first item |
c.month = request.params.get('month', '') | c.month = request.params.get('month', '') |
if not c.month: | if not c.month: |
c.month_desc = 'all months' | c.month_desc = 'all months' |
else: | else: |
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) | c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) |
c.publisher_page_views = 0 | c.publisher_page_views = 0 |
q = model.Session.query(GA_Url).\ | q = model.Session.query(GA_Url).\ |
filter(GA_Url.url=='/publisher/%s' % c.publisher.name) | filter(GA_Url.url=='/publisher/%s' % c.publisher.name) |
if c.month: | if c.month: |
entry = q.filter(GA_Url.period_name==c.month).first() | entry = q.filter(GA_Url.period_name==c.month).first() |
c.publisher_page_views = entry.pageviews if entry else 0 | c.publisher_page_views = entry.pageviews if entry else 0 |
else: | else: |
for e in q.all(): | for e in q.all(): |
c.publisher_page_views = c.publisher_page_views + int(e.pageviews) | c.publisher_page_views = c.publisher_page_views + int(e.pageviews) |
c.top_packages = self._get_packages(c.publisher, 20) | c.top_packages = self._get_packages(c.publisher, 20) |
return render('ga_report/publisher/read.html') | return render('ga_report/publisher/read.html') |
def _get_publishers(limit=20): | |
connection = model.Session.connection() | |
q = """ | |
select department_id, sum(pageviews::int) views, sum(visitors::int) visits | |
from ga_url | |
where department_id <> ''""" | |
if c.month: | |
q = q + """ | |
and period_name=%s | |
""" | |
q = q + """ | |
group by department_id order by views desc | |
""" | |
if limit: | |
q = q + " limit %s;" % (limit) | |
# Add this back (before and period_name =%s) if you want to ignore publisher | |
# homepage views | |
# and not url like '/publisher/%%' | |
top_publishers = [] | |
res = connection.execute(q, c.month) | |
for row in res: | |
g = model.Group.get(row[0]) | |
if g: | |
top_publishers.append((g, row[1], row[2])) | |
return top_publishers | |
import logging | import logging |
import operator | import operator |
import ckan.lib.base as base | import ckan.lib.base as base |
import ckan.model as model | import ckan.model as model |
from ckanext.ga_report.ga_model import GA_Url, GA_Publisher | |
from ckanext.ga_report.controller import _get_publishers | |
_log = logging.getLogger(__name__) | _log = logging.getLogger(__name__) |
def popular_datasets(count=10): | |
import random | |
publisher = None | |
publishers = _get_publishers(30) | |
total = len(publishers) | |
while not publisher or not datasets: | |
rand = random.randrange(0, total) | |
publisher = publishers[rand][0] | |
if not publisher.state == 'active': | |
publisher = None | |
continue | |
datasets = _datasets_for_publisher(publisher, 10)[:count] | |
ctx = { | |
'datasets': datasets, | |
'publisher': publisher | |
} | |
return base.render_snippet('ga_report/ga_popular_datasets.html', **ctx) | |
def single_popular_dataset(top=20): | |
import random | |
datasets = {} | |
rand = random.randrange(0, top) | |
entry = model.Session.query(GA_Url).\ | |
filter(GA_Url.url.like('/dataset/%')).\ | |
order_by('ga_url.pageviews::int desc')[rand] | |
dataset = None | |
while not dataset: | |
dataset = model.Package.get(entry.url[len('/dataset/'):]) | |
if dataset and not dataset.state == 'active': | |
dataset = None | |
else: | |
publisher = model.Group.get(entry.department_id) | |
ctx = { | |
'dataset': dataset, | |
'publisher': publisher | |
} | |
return base.render_snippet('ga_report/ga_popular_single.html', **ctx) | |
def most_popular_datasets(publisher, count=20): | def most_popular_datasets(publisher, count=20): |
from ckanext.ga_report.ga_model import GA_Url | |
if not publisher: | if not publisher: |
_log.error("No valid publisher passed to 'most_popular_datasets'") | _log.error("No valid publisher passed to 'most_popular_datasets'") |
return "" | return "" |
results = _datasets_for_publisher(publisher, count) | |
ctx = { | |
'dataset_count': len(datasets), | |
'datasets': results, | |
'publisher': publisher | |
} | |
return base.render_snippet('ga_report/publisher/popular.html', **ctx) | |
def _datasets_for_publisher(publisher, count): | |
datasets = {} | datasets = {} |
entries = model.Session.query(GA_Url).\ | entries = model.Session.query(GA_Url).\ |
filter(GA_Url.department_id==publisher.name).\ | filter(GA_Url.department_id==publisher.name).\ |
filter(GA_Url.url.like('/dataset/%')).\ | filter(GA_Url.url.like('/dataset/%')).\ |
order_by('ga_url.pageviews::int desc').all() | order_by('ga_url.pageviews::int desc').all() |
for entry in entries: | for entry in entries: |
if len(datasets) < count: | if len(datasets) < count: |
p = model.Package.get(entry.url[len('/dataset/'):]) | p = model.Package.get(entry.url[len('/dataset/'):]) |
if not p in datasets: | if not p in datasets: |
datasets[p] = {'views':0, 'visits': 0} | datasets[p] = {'views':0, 'visits': 0} |
datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) | datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) |
datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) | datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) |
results = [] | results = [] |
for k, v in datasets.iteritems(): | for k, v in datasets.iteritems(): |
results.append((k,v['views'],v['visits'])) | results.append((k,v['views'],v['visits'])) |
results = sorted(results, key=operator.itemgetter(1), reverse=True) | return sorted(results, key=operator.itemgetter(1), reverse=True) |
ctx = { | |
'dataset_count': len(datasets), | |
'datasets': results, | |
'publisher': publisher | |
} | |
return base.render_snippet('ga_report/publisher/popular.html', **ctx) | |
import logging | import logging |
import ckan.lib.helpers as h | import ckan.lib.helpers as h |
import ckan.plugins as p | import ckan.plugins as p |
from ckan.plugins import implements, toolkit | from ckan.plugins import implements, toolkit |
from ckanext.ga_report.helpers import (most_popular_datasets, | |
popular_datasets, | |
single_popular_dataset) | |
log = logging.getLogger('ckanext.ga-report') | log = logging.getLogger('ckanext.ga-report') |
class GAReportPlugin(p.SingletonPlugin): | class GAReportPlugin(p.SingletonPlugin): |
implements(p.IConfigurer, inherit=True) | implements(p.IConfigurer, inherit=True) |
implements(p.IRoutes, inherit=True) | implements(p.IRoutes, inherit=True) |
implements(p.ITemplateHelpers, inherit=True) | implements(p.ITemplateHelpers, inherit=True) |
def update_config(self, config): | def update_config(self, config): |
toolkit.add_template_directory(config, 'templates') | toolkit.add_template_directory(config, 'templates') |
toolkit.add_public_directory(config, 'public') | toolkit.add_public_directory(config, 'public') |
def get_helpers(self): | def get_helpers(self): |
""" | """ |
A dictionary of extra helpers that will be available to provide | A dictionary of extra helpers that will be available to provide |
ga report info to templates. | ga report info to templates. |
""" | """ |
from ckanext.ga_report.helpers import most_popular_datasets | |
return { | return { |
'ga_report_installed': lambda: True, | 'ga_report_installed': lambda: True, |
'popular_datasets': popular_datasets, | |
'most_popular_datasets': most_popular_datasets, | 'most_popular_datasets': most_popular_datasets, |
'single_popular_dataset': single_popular_dataset | |
} | } |
def after_map(self, map): | def after_map(self, map): |
map.connect( | map.connect( |
'/data/site-usage/publisher', | '/data/site-usage/publisher', |
controller='ckanext.ga_report.controller:GaPublisherReport', | controller='ckanext.ga_report.controller:GaPublisherReport', |
action='index' | action='index' |
) | ) |
map.connect( | map.connect( |
'/data/site-usage/publisher_{month}.csv', | '/data/site-usage/publisher_{month}.csv', |
controller='ckanext.ga_report.controller:GaPublisherReport', | controller='ckanext.ga_report.controller:GaPublisherReport', |
action='csv' | action='csv' |
) | ) |
map.connect( | map.connect( |
'/data/site-usage/publisher/{id}_{month}.csv', | '/data/site-usage/publisher/{id}_{month}.csv', |
controller='ckanext.ga_report.controller:GaPublisherReport', | controller='ckanext.ga_report.controller:GaPublisherReport', |
action='publisher_csv' | action='publisher_csv' |
) | ) |
map.connect( | map.connect( |
'/data/site-usage/publisher/{id}', | '/data/site-usage/publisher/{id}', |
controller='ckanext.ga_report.controller:GaPublisherReport', | controller='ckanext.ga_report.controller:GaPublisherReport', |
action='read' | action='read' |
) | ) |
map.connect( | map.connect( |
'/data/site-usage', | '/data/site-usage', |
controller='ckanext.ga_report.controller:GaReport', | controller='ckanext.ga_report.controller:GaReport', |
action='index' | action='index' |
) | ) |
map.connect( | map.connect( |
'/data/site-usage/data_{month}.csv', | '/data/site-usage/data_{month}.csv', |
controller='ckanext.ga_report.controller:GaReport', | controller='ckanext.ga_report.controller:GaReport', |
action='csv' | action='csv' |
) | ) |
return map | return map |
<html xmlns:py="http://genshi.edgewall.org/" | |
xmlns:i18n="http://genshi.edgewall.org/i18n" | |
xmlns:xi="http://www.w3.org/2001/XInclude" | |
py:strip=""> | |
<div class="popular_datasets"> | |
<div class="pull-right"> | |
<a href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" class="btn btn-primary">More popular datasets »</a> | |
</div> | |
<h2>Popular datasets</h2> | |
<h4>${publisher.title}</h4> | |
<ul> | |
<py:for each="dataset, _, _ in datasets"> | |
<li> | |
<span>${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))}</span> | |
<div>${h.truncate(dataset.notes, length=80, whole_word=True)}</div> | |
</li> | |
</py:for> | |
</ul> | |
</div> | |
</html> | |
<html xmlns:py="http://genshi.edgewall.org/" | |
xmlns:i18n="http://genshi.edgewall.org/i18n" | |
xmlns:xi="http://www.w3.org/2001/XInclude" | |
py:strip=""> | |
<div class="popular_datasets"> | |
<h2>Featured dataset</h2> | |
<h3>${h.link_to(dataset.title, h.url_for(controller='package', action='read', id=dataset.name))}</h3> <div>${h.truncate(dataset.notes, length=200, whole_word=True)}</div> | |
<p></p> | |
<div> | |
<a href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" class="btn btn-primary">More popular datasets</a> | |
<a href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}" class="btn btn-primary">All usage data</a> | |
</div> | |
</div> | |
</html> | |
<html | <html |
xmlns="http://www.w3.org/1999/xhtml" | xmlns="http://www.w3.org/1999/xhtml" |
xmlns:i18n="http://genshi.edgewall.org/i18n" | xmlns:i18n="http://genshi.edgewall.org/i18n" |
xmlns:py="http://genshi.edgewall.org/" | xmlns:py="http://genshi.edgewall.org/" |
xmlns:xi="http://www.w3.org/2001/XInclude" | xmlns:xi="http://www.w3.org/2001/XInclude" |
py:strip="" | py:strip="" |
> | > |
<table py:def="publisher_list(groups)" class="groups"> | |
<py:for each="group,title in groups"> | |
<tr> | |
<td><a href="/publisher/${group.name}">${title}</a></td> | |
</tr> | |
</py:for> | |
</table> | |
<table py:def="social_table(items, with_source=False)" class="table table-condensed table-bordered table-striped"> | <table py:def="social_table(items, with_source=False)" class="table table-condensed table-bordered table-striped"> |
<tr> | <tr> |
<th>Name</th> | <th>Name</th> |
<th py:if="with_source">Source</th> | <th py:if="with_source">Source</th> |
<th>Visits</th> | <th>Visits</th> |
</tr> | </tr> |
<py:for each="name, url, source, count in items"> | <py:for each="name, url, source, count in items"> |
<tr> | <tr> |
<td><a href="${url}">${name}</a></td> | <td><a href="${url}">${name}</a></td> |
<td py:if="with_source">${source}</td> | <td py:if="with_source">${source}</td> |
<td>${count}</td> | <td>${count}</td> |
</tr> | </tr> |
</py:for> | </py:for> |
</table> | </table> |
<table py:def="stat_table(items, title='Views')" class="table table-condensed table-bordered table-striped"> | <table py:def="stat_table(items, title='Views')" class="table table-condensed table-bordered table-striped"> |
<tr> | <tr> |
<th>Name</th> | <th>Name</th> |
<th>% ${title}</th> | <th>% ${title}</th> |
</tr> | </tr> |
<py:for each="name, value in items"> | <py:for each="name, value in items"> |
<tr> | <tr> |
<td>${name}</td> | <td>${name}</td> |
<td>${value}</td> | <td>${value}</td> |
</tr> | </tr> |
</py:for> | </py:for> |
</table> | </table> |
<div py:def="usage_nav(active_name,publisher)" id="minornavigation"> | <div py:def="usage_nav(active_name,publisher)" id="minornavigation"> |
<div id="minornavigation-bg-left"> | <div id="minornavigation-bg-left"> |
<div id="minornavigation-bg-right"> | <div id="minornavigation-bg-right"> |
<ul class="nav nav-pills"> | <ul class="nav nav-pills"> |
<li py:attrs="{'class': 'active' if active_name=='Site-wide' else None}"><a py:attrs="{'class': 'active' if active_name=='Site-wide' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/> Site-wide</a></li> | <li py:attrs="{'class': 'active' if active_name=='Site-wide' else None}"><a py:attrs="{'class': 'active' if active_name=='Site-wide' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaReport',action='index')}"><img src="/images/icons/page_white.png" height="16px" width="16px" alt="None" class="inline-icon "/> Site-wide</a></li> |
<li py:attrs="{'class': 'active' if active_name=='Publishers' else None}"> | <li py:attrs="{'class': 'active' if active_name=='Publishers' else None}"> |
<a py:attrs="{'class': 'active' if active_name=='Publishers' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/> Publishers</a> | <a py:attrs="{'class': 'active' if active_name=='Publishers' else None}" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}"><img src="/images/icons/page_white.png" height="16px" width="16px" alt="None" class="inline-icon "/> Publishers</a> |
</li> | </li> |
<li py:if="publisher" class="active"> | <li py:if="publisher" class="active"> |
<a class="active" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read', id=publisher.name)}"><img src="/images/icons/page_white_gear.png" height="16px" width="16px" alt="None" class="inline-icon "/>${publisher.title}</a> | <a class="active" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read', id=publisher.name)}"><img src="/images/icons/page_white.png" height="16px" width="16px" alt="None" class="inline-icon "/>${publisher.title}</a> |
</li> | </li> |
</ul> | </ul> |
</div> | </div> |
</div> | </div> |
</div> | </div> |
</html> | </html> |
<html xmlns:py="http://genshi.edgewall.org/" | <html xmlns:py="http://genshi.edgewall.org/" |
xmlns:i18n="http://genshi.edgewall.org/i18n" | xmlns:i18n="http://genshi.edgewall.org/i18n" |
xmlns:xi="http://www.w3.org/2001/XInclude" | xmlns:xi="http://www.w3.org/2001/XInclude" |
py:strip=""> | py:strip=""> |
<xi:include href="../ga_util.html" /> | <xi:include href="../ga_util.html" /> |
<py:def function="page_title">Publisher Analytics for ${g.site_title}</py:def> | <py:def function="page_title">Publisher Analytics for ${g.site_title}</py:def> |
<py:match path="primarysidebar"> | <py:match path="primarysidebar"> |
<li class="widget-container boxed widget_text"> | <li class="widget-container boxed widget_text"> |
<h4>Publishers</h4> | <h4>Publishers</h4> |
<p> | <p> |
Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. | Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. |
</p> | </p> |
<p> | <p> |
Note: this data does not include API calls. | Note: this data does not include API calls. |
</p> | </p> |
</li> | </li> |
<li class="widget-container boxed widget_text"> | <li class="widget-container boxed widget_text"> |
<h4>Download</h4> | <h4>Download</h4> |
<p><center> | <p><center> |
<a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='csv',month=c.month or 'all')}">Download as CSV</a></center> | <a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='csv',month=c.month or 'all')}">Download as CSV</a></center> |
</p> | </p> |
</li> | </li> |
</py:match> | </py:match> |
<div py:match="content"> | <div py:match="content"> |
<h1>Site Usage</h1> | <h1>Site Usage</h1> |
${usage_nav('Publishers', None)} | ${usage_nav('Publishers', None)} |
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" method="get"> | <form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='index')}" method="get"> |
<div class="controls"> | <div class="controls"> |
<select name="month"> | <select name="month"> |
<option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option> | <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option> |
<py:for each="val,desc in c.months"> | <py:for each="val,desc in c.months"> |
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option> | <option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option> |
</py:for> | </py:for> |
</select> | </select> |
<input class="btn button" type='submit' value="Update"/> | <input class="btn button" type='submit' value="Update"/> |
</div> | </div> |
</form> | </form> |
<table class="table table-condensed table-bordered table-striped"> | <table class="table table-condensed table-bordered table-striped"> |
<tr> | <tr> |
<th>Publisher</th> | <th>Publisher</th> |
<th>Dataset Views</th> | <th>Dataset Views</th> |
<th>Visits</th> | <th>Visits</th> |
</tr> | </tr> |
<py:for each="publisher, views, visits in c.top_publishers"> | <py:for each="publisher, views, visits in c.top_publishers"> |
<tr> | <tr> |
<td>${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport', action='read', id=publisher.name))} | <td>${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport', action='read', id=publisher.name))} |
</td> | </td> |
<td>${views}</td> | <td>${views}</td> |
<td>${visits}</td> | <td>${visits}</td> |
</tr> | </tr> |
</py:for> | </py:for> |
</table> | </table> |
</div> | </div> |
<xi:include href="../../layout.html" /> | <xi:include href="../../layout.html" /> |
<py:def function="optional_footer"> | <py:def function="optional_footer"> |
<script type='text/javascript'> | <script type='text/javascript'> |
$('.nav-tabs li a').click(function (e) { | $('.nav-tabs li a').click(function (e) { |
e.preventDefault(); | e.preventDefault(); |
$(this).tab('show'); | $(this).tab('show'); |
}) | }) |
</script> | </script> |
</py:def> | </py:def> |
</html> | </html> |
<html xmlns:py="http://genshi.edgewall.org/" | <html xmlns:py="http://genshi.edgewall.org/" |
xmlns:i18n="http://genshi.edgewall.org/i18n" | xmlns:i18n="http://genshi.edgewall.org/i18n" |
xmlns:xi="http://www.w3.org/2001/XInclude" | xmlns:xi="http://www.w3.org/2001/XInclude" |
py:strip=""> | py:strip=""> |
<xi:include href="../ga_util.html" /> | <xi:include href="../ga_util.html" /> |
<py:def function="page_title">Analytics for ${g.site_title}</py:def> | <py:def function="page_title">Analytics for ${g.site_title}</py:def> |
<py:match path="primarysidebar"> | <py:match path="primarysidebar"> |
<li class="widget-container boxed widget_text"> | <li class="widget-container boxed widget_text"> |
<h4>Publishers</h4> | <h4>Publishers</h4> |
<p> | <p> |
Dataset views records the number of times a specific dataset page has been viewed. Visits records the number of unique site visits. | Dataset views records the number of times a specific dataset page has been viewed for <a href="${h.url_for(controller='ckanext.dgu.controllers.publisher:PublisherController',action='read',id=c.publisher.name)}">${c.publisher.title}</a>. Visits records the number of unique site visits. |
</p> | </p> |
<p> | <p> |
Note: this data does not include API calls. | Note: this data does not include API calls. |
</p> | </p> |
</li> | </li> |
<li class="widget-container boxed widget_text"> | <li class="widget-container boxed widget_text"> |
<h4>Download</h4> | <h4>Download</h4> |
<p><center> | <p><center> |
<a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='publisher_csv',id=c.publisher.name,month=c.month or 'all')}">Download as CSV</a></center> | <a class="btn button" href="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='publisher_csv',id=c.publisher.name,month=c.month or 'all')}">Download as CSV</a></center> |
</p> | </p> |
</li> | </li> |
</py:match> | </py:match> |
<div py:match="content"> | <div py:match="content"> |
<h1>Site Usage</h1> | <h1>Site Usage</h1> |
${usage_nav(c.publisher.title, c.publisher)} | ${usage_nav(c.publisher.title, c.publisher)} |
<form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=c.publisher.name)}" method="get"> | <form class="form-inline" action="${h.url_for(controller='ckanext.ga_report.controller:GaPublisherReport',action='read',id=c.publisher.name)}" method="get"> |
<div class="controls"> | <div class="controls"> |
<select name="month"> | <select name="month"> |
<option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option> | <option value='' py:attrs="{'selected': 'selected' if not c.month else None}">All months</option> |
<py:for each="val,desc in c.months"> | <py:for each="val,desc in c.months"> |
<option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option> | <option value='${val}' py:attrs="{'selected': 'selected' if c.month == val else None}">${desc}</option> |
</py:for> | </py:for> |
</select> | </select> |
<input class="btn button" type='submit' value="Update"/> | <input class="btn button" type='submit' value="Update"/> |
</div> | </div> |
</form> | </form> |
<table class="table table-condensed table-bordered table-striped"> | <table class="table table-condensed table-bordered table-striped"> |
<tr> | <tr> |
<th>Dataset</th> | <th>Dataset</th> |
<th>Views</th> | <th>Views</th> |
<th>Visits</th> | <th>Visits</th> |
</tr> | </tr> |
<py:for each="package, views, visits in c.top_packages"> | <py:for each="package, views, visits in c.top_packages"> |
<tr> | <tr> |
<td>${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} | <td>${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} |
</td> | </td> |
<td>${views}</td> | <td>${views}</td> |
<td>${visits}</td> | <td>${visits}</td> |
</tr> | </tr> |
</py:for> | </py:for> |
</table> | </table> |
</div> | </div> |
<xi:include href="../../layout.html" /> | <xi:include href="../../layout.html" /> |
</html> | </html> |