Changes to tidy up the handling of all months;
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -9,7 +9,7 @@
import sqlalchemy
from sqlalchemy import func, cast, Integer
import ckan.model as model
-from ga_model import GA_Url, GA_Stat, GA_ReferralStat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher
log = logging.getLogger('ckanext.ga-report')
@@ -228,7 +228,6 @@
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
c.top_publishers = _get_top_publishers()
-
return render('ga_report/publisher/index.html')
def _get_packages(self, publisher=None, count=-1):
@@ -246,7 +245,6 @@
q = q.filter(GA_Url.period_name==month)
q = q.order_by('ga_url.visitors::int desc')
top_packages = []
-
for entry,package in q.limit(count):
if package:
top_packages.append((package, entry.pageviews, entry.visitors))
@@ -288,7 +286,7 @@
else:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
- month = c.mnth or 'All'
+ month = c.month or 'All'
c.publisher_page_views = 0
q = model.Session.query(GA_Url).\
filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
@@ -304,33 +302,26 @@
Returns a list of the top 20 publishers by dataset visits.
(The number to show can be varied with 'limit')
'''
+ month = c.month or 'All'
connection = model.Session.connection()
q = """
select department_id, sum(pageviews::int) views, sum(visitors::int) visits
from ga_url
- where department_id <> ''"""
- if c.month:
- q = q + """
- and period_name=%s
- """
- q = q + """
- group by department_id order by visits desc
+ where department_id <> ''
+ and period_name=%s
+ group by department_id order by visits desc
"""
if limit:
q = q + " limit %s;" % (limit)
- # Add this back (before and period_name =%s) if you want to ignore publisher
- # homepage views
- # and not url like '/publisher/%%'
-
top_publishers = []
- res = connection.execute(q, c.month)
-
+ res = connection.execute(q, month)
for row in res:
g = model.Group.get(row[0])
if g:
top_publishers.append((g, row[1], row[2]))
return top_publishers
+
def _get_publishers():
'''
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -3,7 +3,7 @@
import datetime
import collections
from pylons import config
-
+from ga_model import _normalize_url
import ga_model
#from ga_client import GA
@@ -101,12 +101,17 @@
start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d'))
- data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
+ # Clean up the entries before we run this
+ ga_model.pre_update_url_stats(period_name)
+
+ accountName = config.get('googleanalytics.account')
+
+ data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName)
log.info('Storing Dataset Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day))
self.store(period_name, period_complete_day, data, )
- data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
+ data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName)
log.info('Storing Publisher Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day))
self.store(period_name, period_complete_day, data,)
@@ -115,6 +120,7 @@
self.sitewide_stats( period_name )
self.update_social_info(period_name, start_date, end_date)
+
def update_social_info(self, period_name, start_date, end_date):
start_date = start_date.strftime('%Y-%m-%d')
@@ -137,12 +143,11 @@
data = collections.defaultdict(list)
rows = results.get('rows',[])
for row in rows:
- from ga_model import _normalize_url
data[_normalize_url(row[0])].append( (row[1], int(row[2]),) )
ga_model.update_social(period_name, data)
- def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
+ def download(self, start_date, end_date, path=None):
'''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
@@ -165,7 +170,10 @@
packages = []
for entry in results.get('rows'):
(loc,pageviews,visits) = entry
- packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
+ url = _normalize_url('http:/' + loc)
+ if not url.startswith('/dataset/') and not url.startswith('/publisher/'):
+ continue
+ packages.append( (url, pageviews, visits,) ) # Temporary hack
return dict(url=packages)
def store(self, period_name, period_complete_day, data):
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -156,46 +156,9 @@
model.Session.commit()
-
-def update_url_stats(period_name, period_complete_day, url_data):
- for url, views, visitors in url_data:
- url = _normalize_url(url)
- department_id = _get_department_id_of_url(url)
-
- package = None
- if url.startswith('/dataset/'):
- package = url[len('/dataset/'):]
-
- # see if the row for this url & month is in the table already
- item = model.Session.query(GA_Url).\
- filter(GA_Url.period_name==period_name).\
- filter(GA_Url.url==url).first()
- if item:
- item.period_name = period_name
- item.pageviews = views
- item.visitors = visitors
- item.department_id = department_id
- item.package_id = package
- model.Session.add(item)
- else:
- # create the row
- values = {'id': make_uuid(),
- 'period_name': period_name,
- 'period_complete_day': period_complete_day,
- 'url': url,
- 'pageviews': views,
- 'visitors': visitors,
- 'department_id': department_id,
- 'package_id': package
- }
- model.Session.add(GA_Url(**values))
-
- # We now need to recaculate the ALL time_period from the data we have
- # Delete the old 'All'
- old = model.Session.query(GA_Url).\
- filter(GA_Url.period_name == "All").\
- filter(GA_Url.url==url).delete()
-
+def update_url_stat_totals(period_name):
+
+ """
items = model.Session.query(GA_Url).\
filter(GA_Url.period_name != "All").\
filter(GA_Url.url==url).all()
@@ -209,8 +172,55 @@
'package_id': package
}
model.Session.add(GA_Url(**values))
-
model.Session.commit()
+ """
+
+def pre_update_url_stats(period_name):
+ model.Session.query(GA_Url).\
+ filter(GA_Url.period_name==period_name).delete()
+ model.Session.query(GA_Url).\
+ filter(GA_Url.period_name=='All').delete()
+
+
+def update_url_stats(period_name, period_complete_day, url_data):
+
+ for url, views, visitors in url_data:
+ department_id = _get_department_id_of_url(url)
+
+ package = None
+ if url.startswith('/dataset/'):
+ package = url[len('/dataset/'):]
+
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'period_complete_day': period_complete_day,
+ 'url': url,
+ 'pageviews': views,
+ 'visitors': visitors,
+ 'department_id': department_id,
+ 'package_id': package
+ }
+ model.Session.add(GA_Url(**values))
+ model.Session.commit()
+
+ if package:
+ entries = model.Session.query(GA_Url).\
+ filter(GA_Url.period_name!='All').\
+ filter(GA_Url.url==url).all()
+ values = {'id': make_uuid(),
+ 'period_name': 'All',
+ 'period_complete_day': 0,
+ 'url': url,
+ 'pageviews': sum([int(e.pageviews) for e in entries]),
+ 'visitors': sum([int(e.visitors) for e in entries]),
+ 'department_id': department_id,
+ 'package_id': package
+ }
+ model.Session.add(GA_Url(**values))
+ model.Session.commit()
+
+
+
def update_social(period_name, data):
--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -41,15 +41,15 @@
<table class="table table-condensed table-bordered table-striped">
<tr>
<th>Publisher</th>
+ <th>Dataset Visits</th>
<th>Dataset Views</th>
- <th>Dataset Visits</th>
</tr>
<py:for each="publisher, views, visits in c.top_publishers">
<tr>
<td>${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name))}
</td>
+ <td>${visits}</td>
<td>${views}</td>
- <td>${visits}</td>
</tr>
</py:for>
</table>
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -47,15 +47,15 @@
<table py:if="c.top_packages" class="table table-condensed table-bordered table-striped">
<tr>
<th>Dataset</th>
+ <th>Visits</th>
<th>Views</th>
- <th>Visits</th>
</tr>
<py:for each="package, views, visits in c.top_packages">
<tr>
<td>${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}
</td>
+ <td>${visits}</td>
<td>${views}</td>
- <td>${visits}</td>
</tr>
</py:for>
</table>