Changed to pageviews instead of unique pageviews
--- a/README.rst
+++ b/README.rst
@@ -33,9 +33,10 @@
googleanalytics.id = UA-1010101-1
googleanalytics.account = Account name (e.g. data.gov.uk, see top level item at https://www.google.com/analytics)
ga-report.period = monthly
- ga-report.bounce_url = /
+ ga-report.bounce_url = /data
- The ga-report.bounce_url specifies a particular path to record the bounce rate for. Typically it is / (the home page).
+ The ga-report.bounce_url specifies the path to use when calculating bounces. For DGU this is /data
+ but you may want to set this to /.
3. Set up this extension's database tables using a paster command. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file)::
--- a/ckanext/ga_report/command.py
+++ b/ckanext/ga_report/command.py
@@ -80,11 +80,6 @@
default=False,
dest='delete_first',
help='Delete data for the period first')
- self.parser.add_option('-s', '--slip_url_stats',
- action='store_true',
- default=False,
- dest='skip_url_stats',
- help='Skip the download of URL data - just do site-wide stats')
def command(self):
self._load_config()
@@ -100,8 +95,7 @@
return
downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc),
- delete_first=self.options.delete_first,
- skip_url_stats=self.options.skip_url_stats)
+ delete_first=self.options.delete_first)
time_period = self.args[1] if self.args and len(self.args) > 1 \
else 'latest'
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -244,11 +244,11 @@
if publisher:
q = q.filter(GA_Url.department_id==publisher.name)
q = q.filter(GA_Url.period_name==month)
- q = q.order_by('ga_url.visitors::int desc')
+ q = q.order_by('ga_url.visits::int desc')
top_packages = []
for entry,package in q.limit(count):
if package:
- top_packages.append((package, entry.pageviews, entry.visitors))
+ top_packages.append((package, entry.pageviews, entry.visits))
else:
log.warning('Could not find package associated package')
@@ -306,7 +306,7 @@
month = c.month or 'All'
connection = model.Session.connection()
q = """
- select department_id, sum(pageviews::int) views, sum(visitors::int) visits
+ select department_id, sum(pageviews::int) views, sum(visits::int) visits
from ga_url
where department_id <> ''
and package_id <> ''
--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -17,13 +17,11 @@
class DownloadAnalytics(object):
'''Downloads and stores analytics info'''
- def __init__(self, service=None, profile_id=None, delete_first=False,
- skip_url_stats=False):
+ def __init__(self, service=None, profile_id=None, delete_first=False):
self.period = config['ga-report.period']
self.service = service
self.profile_id = profile_id
self.delete_first = delete_first
- self.skip_url_stats = skip_url_stats
def specific_month(self, date):
import calendar
@@ -98,32 +96,31 @@
self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
-
+
if self.delete_first:
log.info('Deleting existing Analytics for this period "%s"',
period_name)
ga_model.delete(period_name)
- if not self.skip_url_stats:
- # Clean out old url data before storing the new
- ga_model.pre_update_url_stats(period_name)
-
- accountName = config.get('googleanalytics.account')
-
- log.info('Downloading analytics for dataset views')
- data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName)
-
- log.info('Storing dataset views (%i rows)', len(data.get('url')))
- self.store(period_name, period_complete_day, data, )
-
- log.info('Downloading analytics for publisher views')
- data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName)
-
- log.info('Storing publisher views (%i rows)', len(data.get('url')))
- self.store(period_name, period_complete_day, data,)
-
- log.info('Aggregating datasets by publisher')
- ga_model.update_publisher_stats(period_name) # about 30 seconds.
+ # Clean up the entries before we run this
+ ga_model.pre_update_url_stats(period_name)
+
+ accountName = config.get('googleanalytics.account')
+
+ log.info('Downloading analytics for dataset views')
+ data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName)
+
+ log.info('Storing dataset views (%i rows)', len(data.get('url')))
+ self.store(period_name, period_complete_day, data, )
+
+ log.info('Downloading analytics for publisher views')
+ data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName)
+
+ log.info('Storing publisher views (%i rows)', len(data.get('url')))
+ self.store(period_name, period_complete_day, data,)
+
+ log.info('Aggregating datasets by publisher')
+ ga_model.update_publisher_stats(period_name) # about 30 seconds.
log.info('Downloading and storing analytics for site-wide stats')
self.sitewide_stats( period_name )
@@ -162,8 +159,8 @@
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:pagePath=%s$' % path
- metrics = 'ga:uniquePageviews, ga:visits'
- sort = '-ga:uniquePageviews'
+ metrics = 'ga:pageviews, ga:visits'
+ sort = '-ga:pageviews'
# Supported query params at
# https://developers.google.com/analytics/devguides/reporting/core/v3/reference
@@ -180,12 +177,8 @@
packages = []
for entry in results.get('rows'):
(loc,pageviews,visits) = entry
- url = _normalize_url('http:/' + loc) # strips off domain e.g. www.data.gov.uk or data.gov.uk
-
+ url = _normalize_url('http:/' + loc)
if not url.startswith('/dataset/') and not url.startswith('/publisher/'):
- # filter out strays like:
- # /data/user/login?came_from=http://data.gov.uk/dataset/os-code-point-open
- # /403.html?page=/about&from=http://data.gov.uk/publisher/planning-inspectorate
continue
packages.append( (url, pageviews, visits,) ) # Temporary hack
return dict(url=packages)
@@ -219,8 +212,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
@@ -241,27 +234,25 @@
}
ga_model.update_sitewide_stats(period_name, "Totals", data)
- # Bounces from / or another configurable page.
- path = '/%s%s' % (config.get('googleanalytics.account'),
- config.get('ga-report.bounce_url', '/'))
- results = self.service.data().ga().get(
- ids='ga:' + self.profile_id,
- filters='ga:pagePath==%s' % (path,),
- start_date=start_date,
- metrics='ga:bounces,ga:uniquePageviews',
+ # Bounces from /data. This url is specified in configuration because
+ # for DGU we don't want /.
+ path = config.get('ga-report.bounce_url','/')
+ print path
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ filters='ga:pagePath=~%s$' % (path,),
+ start_date=start_date,
+ metrics='ga:bounces,ga:pageviews',
dimensions='ga:pagePath',
max_results=10000,
end_date=end_date).execute()
result_data = results.get('rows')
- if len(result_data) != 1:
- log.error('Could not pinpoint the bounces for path: %s. Got results: %r',
- path, result_data)
- return
- results = result_data[0]
- bounces, total = [float(x) for x in result_data[0][1:]]
- pct = 100 * bounces/total
- log.info('%d bounces from %d total == %s', bounces, total, pct)
- ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate': pct})
+ for results in result_data:
+ if results[0] == path:
+ bounce, total = [float(x) for x in results[1:]]
+ pct = 100 * bounce/total
+ print "%d bounces from %d total == %s" % (bounce, total, pct)
+ ga_model.update_sitewide_stats(period_name, "Totals", {'Bounces': pct})
def _locale_stats(self, start_date, end_date, period_name):
@@ -269,8 +260,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:language,ga:country",
max_results=10000,
end_date=end_date).execute()
@@ -293,8 +284,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:socialNetwork,ga:referralPath",
max_results=10000,
end_date=end_date).execute()
@@ -312,8 +303,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:operatingSystem,ga:operatingSystemVersion",
max_results=10000,
end_date=end_date).execute()
@@ -337,8 +328,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:browser,ga:browserVersion",
max_results=10000,
end_date=end_date).execute()
@@ -386,8 +377,8 @@
results = self.service.data().ga().get(
ids='ga:' + self.profile_id,
start_date=start_date,
- metrics='ga:uniquePageviews',
- sort='-ga:uniquePageviews',
+ metrics='ga:pageviews',
+ sort='-ga:pageviews',
dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo",
max_results=10000,
end_date=end_date).execute()
--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -27,7 +27,7 @@
Column('period_name', types.UnicodeText),
Column('period_complete_day', types.Integer),
Column('pageviews', types.UnicodeText),
- Column('visitors', types.UnicodeText),
+ Column('visits', types.UnicodeText),
Column('url', types.UnicodeText),
Column('department_id', types.UnicodeText),
Column('package_id', types.UnicodeText),
@@ -63,7 +63,7 @@
Column('period_name', types.UnicodeText),
Column('publisher_name', types.UnicodeText),
Column('views', types.UnicodeText),
- Column('visitors', types.UnicodeText),
+ Column('visits', types.UnicodeText),
Column('toplevel', types.Boolean, default=False),
Column('subpublishercount', types.Integer, default=0),
Column('parent', types.UnicodeText),
@@ -111,7 +111,9 @@
>>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices')
'/dataset/weekly_fuel_prices'
'''
- return '/' + '/'.join(url.split('/')[3:])
+ # Deliberately leaving a /
+ url = url.replace('http:/','')
+ return '/' + '/'.join(url.split('/')[2:])
def _get_package_and_publisher(url):
@@ -155,25 +157,6 @@
model.Session.commit()
-def update_url_stat_totals(period_name):
-
- """
- items = model.Session.query(GA_Url).\
- filter(GA_Url.period_name != "All").\
- filter(GA_Url.url==url).all()
- values = {'id': make_uuid(),
- 'period_name': "All",
- 'period_complete_day': "0",
- 'url': url,
- 'pageviews': sum([int(x.pageviews) for x in items]),
- 'visitors': sum([int(x.visitors) for x in items]),
- 'department_id': department_id,
- 'package_id': package
- }
- model.Session.add(GA_Url(**values))
- model.Session.commit()
- """
-
def pre_update_url_stats(period_name):
model.Session.query(GA_Url).\
filter(GA_Url.period_name==period_name).delete()
@@ -182,12 +165,8 @@
def update_url_stats(period_name, period_complete_day, url_data):
- '''
- Given a list of urls and number of hits for each during a given period,
- stores them in GA_Url under the period and recalculates the totals for
- the 'All' period.
- '''
- for url, views, visitors in url_data:
+
+ for url, views, visits in url_data:
package, publisher = _get_package_and_publisher(url)
@@ -196,7 +175,7 @@
filter(GA_Url.url==url).first()
if item:
item.pageviews = item.pageviews + views
- item.visitors = item.visitors + visitors
+ item.visits = item.visits + visits
if not item.package_id:
item.package_id = package
if not item.department_id:
@@ -208,7 +187,7 @@
'period_complete_day': period_complete_day,
'url': url,
'pageviews': views,
- 'visitors': visitors,
+ 'visits': visits,
'department_id': publisher,
'package_id': package
}
@@ -221,7 +200,7 @@
filter(GA_Url.period_name=='All').\
filter(GA_Url.url==url).all()
old_pageviews = sum([int(o.pageviews) for o in old])
- old_visits = sum([int(o.visitors) for o in old])
+ old_visits = sum([int(o.visits) for o in old])
entries = model.Session.query(GA_Url).\
filter(GA_Url.period_name!='All').\
@@ -231,7 +210,7 @@
'period_complete_day': 0,
'url': url,
'pageviews': sum([int(e.pageviews) for e in entries]) + old_pageviews,
- 'visitors': sum([int(e.visitors) for e in entries]) + old_visits,
+ 'visits': sum([int(e.visits) for e in entries]) + old_visits,
'department_id': publisher,
'package_id': package
}
@@ -281,7 +260,7 @@
filter(model.Group.type=='publisher').\
filter(model.Group.state=='active').all()
for publisher in publishers:
- views, visitors, subpub = update_publisher(period_name, publisher, publisher.name)
+ views, visits, subpub = update_publisher(period_name, publisher, publisher.name)
parent, parents = '', publisher.get_groups('publisher')
if parents:
parent = parents[0].name
@@ -290,7 +269,7 @@
filter(GA_Publisher.publisher_name==publisher.name).first()
if item:
item.views = views
- item.visitors = visitors
+ item.visits = visits
item.publisher_name = publisher.name
item.toplevel = publisher in toplevel
item.subpublishercount = subpub
@@ -302,7 +281,7 @@
'period_name': period_name,
'publisher_name': publisher.name,
'views': views,
- 'visitors': visitors,
+ 'visits': visits,
'toplevel': publisher in toplevel,
'subpublishercount': subpub,
'parent': parent
@@ -312,7 +291,7 @@
def update_publisher(period_name, pub, part=''):
- views,visitors,subpub = 0, 0, 0
+ views,visits,subpub = 0, 0, 0
for publisher in go_down_tree(pub):
subpub = subpub + 1
items = model.Session.query(GA_Url).\
@@ -320,9 +299,9 @@
filter(GA_Url.department_id==publisher.name).all()
for item in items:
views = views + int(item.pageviews)
- visitors = visitors + int(item.visitors)
-
- return views, visitors, (subpub-1)
+ visits = visits + int(item.visits)
+
+ return views, visits, (subpub-1)
def get_top_level():
--- a/ckanext/ga_report/helpers.py
+++ b/ckanext/ga_report/helpers.py
@@ -60,8 +60,7 @@
if not dataset:
return None
dataset_dict = get_action('package_show')({'model': model,
- 'session': model.Session,
- 'validate': False},
+ 'session': model.Session},
{'id':dataset.id})
return dataset_dict
@@ -106,7 +105,7 @@
if not p in datasets:
datasets[p] = {'views':0, 'visits': 0}
datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews)
- datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors)
+ datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visits)
results = []
for k, v in datasets.iteritems():
--- a/ckanext/ga_report/templates/ga_report/notes.html
+++ b/ckanext/ga_report/templates/ga_report/notes.html
@@ -8,7 +8,6 @@
<ul>
<li>"Views" is the number of sessions during which the page was viewed one or more times (technically known as "unique pageviews").</li>
<li>"Visits" is the number of unique user visits to a page, counted once for each visitor for each session.</li>
-<!--! <li>"Visitors" is the number of unique users visiting the site (whether once or more times).</li> -->
<li>These usage statistics are confined to users with javascript enabled, which excludes web crawlers and API calls.</li>
<li>The results are not shown when the number of views/visits is tiny. Where these relate to site pages, results are available in full in the CSV download. Where these relate to users' web browser information, results are not disclosed, for privacy reasons.</li>
</ul>
--- a/ckanext/ga_report/tests/test_model.py
+++ /dev/null
@@ -1,18 +1,1 @@
-from nose.tools import assert_equal
-from ckanext.ga_report.ga_model import _normalize_url
-
-class TestNormalizeUrl:
- def test_normal(self):
- assert_equal(_normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices'),
- '/dataset/weekly_fuel_prices')
-
- def test_www_dot(self):
- assert_equal(_normalize_url('http://www.data.gov.uk/dataset/weekly_fuel_prices'),
- '/dataset/weekly_fuel_prices')
-
- def test_https(self):
- assert_equal(_normalize_url('https://data.gov.uk/dataset/weekly_fuel_prices'),
- '/dataset/weekly_fuel_prices')
-
-