Changed to pageviews instead of unique pageviews
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -9,7 +9,7 @@
import sqlalchemy
from sqlalchemy import func, cast, Integer
import ckan.model as model
-from ga_model import GA_Url, GA_Stat, GA_ReferralStat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher
log = logging.getLogger('ckanext.ga-report')
@@ -229,7 +229,6 @@
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
c.top_publishers = _get_top_publishers()
-
return render('ga_report/publisher/index.html')
def _get_packages(self, publisher=None, count=-1):
@@ -245,12 +244,11 @@
if publisher:
q = q.filter(GA_Url.department_id==publisher.name)
q = q.filter(GA_Url.period_name==month)
- q = q.order_by('ga_url.visitors::int desc')
+ q = q.order_by('ga_url.visits::int desc')
top_packages = []
-
for entry,package in q.limit(count):
if package:
- top_packages.append((package, entry.pageviews, entry.visitors))
+ top_packages.append((package, entry.pageviews, entry.visits))
else:
log.warning('Could not find package associated package')
@@ -289,7 +287,7 @@
else:
c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
- month = c.mnth or 'All'
+ month = c.month or 'All'
c.publisher_page_views = 0
q = model.Session.query(GA_Url).\
filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
@@ -305,33 +303,28 @@
Returns a list of the top 20 publishers by dataset visits.
(The number to show can be varied with 'limit')
'''
+ month = c.month or 'All'
connection = model.Session.connection()
q = """
- select department_id, sum(pageviews::int) views, sum(visitors::int) visits
+ select department_id, sum(pageviews::int) views, sum(visits::int) visits
from ga_url
- where department_id <> ''"""
- if c.month:
- q = q + """
- and period_name=%s
- """
- q = q + """
- group by department_id order by visits desc
+ where department_id <> ''
+ and package_id <> ''
+ and url like '/dataset/%%'
+ and period_name=%s
+ group by department_id order by visits desc
"""
if limit:
q = q + " limit %s;" % (limit)
- # Add this back (before and period_name =%s) if you want to ignore publisher
- # homepage views
- # and not url like '/publisher/%%'
-
top_publishers = []
- res = connection.execute(q, c.month)
-
+ res = connection.execute(q, month)
for row in res:
g = model.Group.get(row[0])
if g:
top_publishers.append((g, row[1], row[2]))
return top_publishers
+
def _get_publishers():
'''
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -3,7 +3,7 @@
import datetime
import collections
from pylons import config
-
+from ga_model import _normalize_url
import ga_model
#from ga_client import GA
@@ -96,20 +96,26 @@
self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
-
+
if self.delete_first:
log.info('Deleting existing Analytics for this period "%s"',
period_name)
ga_model.delete(period_name)
+ # Clean up the entries before we run this
+ ga_model.pre_update_url_stats(period_name)
+
+ accountName = config.get('googleanalytics.account')
+
log.info('Downloading analytics for dataset views')
- data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
+ data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName)
log.info('Storing dataset views (%i rows)', len(data.get('url')))
self.store(period_name, period_complete_day, data, )
log.info('Downloading analytics for publisher views')
- data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
+ data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName)
+
log.info('Storing publisher views (%i rows)', len(data.get('url')))
self.store(period_name, period_complete_day, data,)
@@ -121,6 +127,7 @@
log.info('Downloading and storing analytics for social networks')
self.update_social_info(period_name, start_date, end_date)
+
def update_social_info(self, period_name, start_date, end_date):
start_date = start_date.strftime('%Y-%m-%d')
@@ -143,18 +150,17 @@
data = collections.defaultdict(list)
rows = results.get('rows',[])
for row in rows:
- from ga_model import _normalize_url
data[_normalize_url(row[0])].append( (row[1], int(row[2]),) )
ga_model.update_social(period_name, data)
- def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
+ def download(self, start_date, end_date, path=None):
'''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:pagePath=%s$' % path
- metrics = 'ga:uniquePageviews, ga:visits'
- sort = '-ga:uniquePageviews'
+ metrics = 'ga:pageviews, ga:visits'
+ sort = '-ga:pageviews'
# Supported query params at
# https://developers.google.com/analytics/devguides/reporting/core/v3/reference
@@ -171,7 +177,10 @@
packages = []
for entry in results.get('rows'):
(loc,pageviews,visits) = entry
- packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
+ url = _normalize_url('http:/' + loc)
+ if not url.startswith('/dataset/') and not url.startswith('/publisher/'):
+ continue
+ packages.append( (url, pageviews, visits,) ) # Temporary hack
return dict(url=packages)
def store(self, period_name, period_complete_day, data):
@@ -203,8 +212,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
@@ -233,7 +242,7 @@
ids='ga:' + self.profile_id,
filters='ga:pagePath=~%s$' % (path,),
start_date=start_date,
- metrics='ga:bounces,ga:uniquePageviews',
+ metrics='ga:bounces,ga:pageviews',
dimensions='ga:pagePath',
max_results=10000,
end_date=end_date).execute()
@@ -251,8 +260,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:language,ga:country",
max_results=10000,
end_date=end_date).execute()
@@ -275,8 +284,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:socialNetwork,ga:referralPath",
max_results=10000,
end_date=end_date).execute()
@@ -294,8 +303,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:operatingSystem,ga:operatingSystemVersion",
max_results=10000,
end_date=end_date).execute()
@@ -319,8 +328,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:browser,ga:browserVersion",
max_results=10000,
end_date=end_date).execute()
@@ -368,8 +377,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",
max_results=10000,
end_date=end_date).execute()
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -27,7 +27,7 @@
Column('period_name', types.UnicodeText),
Column('period_complete_day', types.Integer),
Column('pageviews', types.UnicodeText),
- Column('visitors', types.UnicodeText),
+ Column('visits', types.UnicodeText),
Column('url', types.UnicodeText),
Column('department_id', types.UnicodeText),
Column('package_id', types.UnicodeText),
@@ -63,7 +63,7 @@
Column('period_name', types.UnicodeText),
Column('publisher_name', types.UnicodeText),
Column('views', types.UnicodeText),
- Column('visitors', types.UnicodeText),
+ Column('visits', types.UnicodeText),
Column('toplevel', types.Boolean, default=False),
Column('subpublishercount', types.Integer, default=0),
Column('parent', types.UnicodeText),
@@ -116,7 +116,7 @@
return '/' + '/'.join(url.split('/')[2:])
-def _get_department_id_of_url(url):
+def _get_package_and_publisher(url):
# e.g. /dataset/fuel_prices
# e.g. /dataset/fuel_prices/resource/e63380d4
dataset_match = re.match('/dataset/([^/]+)(/.*)?', url)
@@ -126,12 +126,13 @@
if dataset:
publisher_groups = dataset.get_groups('publisher')
if publisher_groups:
- return publisher_groups[0].name
+ return dataset_ref,publisher_groups[0].name
+ return dataset_ref, None
else:
publisher_match = re.match('/publisher/([^/]+)(/.*)?', url)
if publisher_match:
- return publisher_match.groups()[0]
-
+ return None, publisher_match.groups()[0]
+ return None, None
def update_sitewide_stats(period_name, stat_name, data):
for k,v in data.iteritems():
@@ -156,61 +157,68 @@
model.Session.commit()
+def pre_update_url_stats(period_name):
+ model.Session.query(GA_Url).\
+ filter(GA_Url.period_name==period_name).delete()
+ model.Session.query(GA_Url).\
+ filter(GA_Url.period_name=='All').delete()
+
def update_url_stats(period_name, period_complete_day, url_data):
- for url, views, visitors in url_data:
- url = _normalize_url(url)
- department_id = _get_department_id_of_url(url)
-
- package = None
- if url.startswith('/dataset/'):
- package = url[len('/dataset/'):]
-
- # see if the row for this url & month is in the table already
+
+ for url, views, visits in url_data:
+ package, publisher = _get_package_and_publisher(url)
+
+
item = model.Session.query(GA_Url).\
filter(GA_Url.period_name==period_name).\
filter(GA_Url.url==url).first()
if item:
- item.period_name = period_name
- item.pageviews = views
- item.visitors = visitors
- item.department_id = department_id
- item.package_id = package
+ item.pageviews = item.pageviews + views
+ item.visits = item.visits + visits
+ if not item.package_id:
+ item.package_id = package
+ if not item.department_id:
+ item.department_id = publisher
model.Session.add(item)
else:
- # create the row
values = {'id': make_uuid(),
'period_name': period_name,
'period_complete_day': period_complete_day,
'url': url,
'pageviews': views,
- 'visitors': visitors,
- 'department_id': department_id,
+ 'visits': visits,
+ 'department_id': publisher,
'package_id': package
}
model.Session.add(GA_Url(**values))
-
- # We now need to recaculate the ALL time_period from the data we have
- # Delete the old 'All'
- old = model.Session.query(GA_Url).\
- filter(GA_Url.period_name == "All").\
- filter(GA_Url.url==url).delete()
-
- items = model.Session.query(GA_Url).\
- filter(GA_Url.period_name != "All").\
- filter(GA_Url.url==url).all()
- values = {'id': make_uuid(),
- 'period_name': "All",
- 'period_complete_day': "0",
- 'url': url,
- 'pageviews': sum([int(x.pageviews) for x in items]),
- 'visitors': sum([int(x.visitors) for x in items]),
- 'department_id': department_id,
- 'package_id': package
- }
- model.Session.add(GA_Url(**values))
-
model.Session.commit()
+
+ if package:
+ old_pageviews, old_visits = 0, 0
+ old = model.Session.query(GA_Url).\
+ filter(GA_Url.period_name=='All').\
+ filter(GA_Url.url==url).all()
+ old_pageviews = sum([int(o.pageviews) for o in old])
+ old_visits = sum([int(o.visits) for o in old])
+
+ entries = model.Session.query(GA_Url).\
+ filter(GA_Url.period_name!='All').\
+ filter(GA_Url.url==url).all()
+ values = {'id': make_uuid(),
+ 'period_name': 'All',
+ 'period_complete_day': 0,
+ 'url': url,
+ 'pageviews': sum([int(e.pageviews) for e in entries]) + old_pageviews,
+ 'visits': sum([int(e.visits) for e in entries]) + old_visits,
+ 'department_id': publisher,
+ 'package_id': package
+ }
+
+ model.Session.add(GA_Url(**values))
+ model.Session.commit()
+
+
def update_social(period_name, data):
@@ -252,7 +260,7 @@
filter(model.Group.type=='publisher').\
filter(model.Group.state=='active').all()
for publisher in publishers:
- views, visitors, subpub = update_publisher(period_name, publisher, publisher.name)
+ views, visits, subpub = update_publisher(period_name, publisher, publisher.name)
parent, parents = '', publisher.get_groups('publisher')
if parents:
parent = parents[0].name
@@ -261,7 +269,7 @@
filter(GA_Publisher.publisher_name==publisher.name).first()
if item:
item.views = views
- item.visitors = visitors
+ item.visits = visits
item.publisher_name = publisher.name
item.toplevel = publisher in toplevel
item.subpublishercount = subpub
@@ -273,7 +281,7 @@
'period_name': period_name,
'publisher_name': publisher.name,
'views': views,
- 'visitors': visitors,
+ 'visits': visits,
'toplevel': publisher in toplevel,
'subpublishercount': subpub,
'parent': parent
@@ -283,7 +291,7 @@
def update_publisher(period_name, pub, part=''):
- views,visitors,subpub = 0, 0, 0
+ views,visits,subpub = 0, 0, 0
for publisher in go_down_tree(pub):
subpub = subpub + 1
items = model.Session.query(GA_Url).\
@@ -291,9 +299,9 @@
filter(GA_Url.department_id==publisher.name).all()
for item in items:
views = views + int(item.pageviews)
- visitors = visitors + int(item.visitors)
-
- return views, visitors, (subpub-1)
+ visits = visits + int(item.visits)
+
+ return views, visits, (subpub-1)
def get_top_level():
--- a/ckanext/ga_report/helpers.py
+++ b/ckanext/ga_report/helpers.py
@@ -105,7 +105,7 @@
if not p in datasets:
datasets[p] = {'views':0, 'visits': 0}
datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
- datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
+ datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visits)
results = []
for k, v in datasets.iteritems():
--- a/ckanext/ga_report/templates/ga_report/notes.html
+++ b/ckanext/ga_report/templates/ga_report/notes.html
@@ -8,7 +8,6 @@
<ul>
<li>"Views" is the number of sessions during which the page was viewed one or more times (technically known as "unique pageviews").</li>
<li>"Visits" is the number of unique user visits to a page, counted once for each visitor for each session.</li>
-<!--! <li>"Visitors" is the number of unique users visiting the site (whether once or more times).</li> -->
<li>These usage statistics are confined to users with javascript enabled, which excludes web crawlers and API calls.</li>
<li>The results are not shown when the number of views/visits is tiny. Where these relate to site pages, results are available in full in the CSV download. Where these relate to users' web browser information, results are not disclosed, for privacy reasons.</li>
</ul>
--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -41,15 +41,15 @@
<table class="table table-condensed table-bordered table-striped">
<tr>
<th>Publisher</th>
+ <th>Dataset Visits</th>
<th>Dataset Views</th>
- <th>Dataset Visits</th>
</tr>
<py:for each="publisher, views, visits in c.top_publishers">
<tr>
<td>${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name))}
</td>
+ <td>${visits}</td>
<td>${views}</td>
- <td>${visits}</td>
</tr>
</py:for>
</table>
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -47,15 +47,15 @@
<table py:if="c.top_packages" class="table table-condensed table-bordered table-striped">
<tr>
<th>Dataset</th>
+ <th>Visits</th>
<th>Views</th>
- <th>Visits</th>
</tr>
<py:for each="package, views, visits in c.top_packages">
<tr>
<td>${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}
</td>
+ <td>${visits}</td>
<td>${views}</td>
- <td>${visits}</td>
</tr>
</py:for>
</table>