From: Tom Rees Date: Fri, 25 Jan 2013 11:36:22 +0000 Subject: #167 Togglable graph legend. Disabled mouseover. X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=e4a17dd849ff955b4d699056f34a39f96203f7b4 --- #167 Togglable graph legend. Disabled mouseover. --- --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.py[co] *.py~ .gitignore +ckan.log # Packages *.egg --- a/README.rst +++ b/README.rst @@ -26,16 +26,17 @@ 1. Activate you CKAN python environment and install this extension's software:: $ pyenv/bin/activate - $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report + $ pip install -e git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration:: googleanalytics.id = UA-1010101-1 - googleanalytics.username = googleaccount@gmail.com - googleanalytics.password = googlepassword + googleanalytics.account = Account name (e.g. data.gov.uk, see top level item at https://www.google.com/analytics) + googleanalytics.token.filepath = ~/pyenv/token.dat ga-report.period = monthly + ga-report.bounce_url = / - Note that your password will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose. + The ga-report.bounce_url specifies a particular path to record the bounce rate for. Typically it is / (the home page). 3. Set up this extension's database tables using a paster command. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file):: @@ -44,6 +45,12 @@ 4. Enable the extension in your CKAN config file by adding it to ``ckan.plugins``:: ckan.plugins = ga-report + +Problem shooting +---------------- + +* ``(ProgrammingError) relation "ga_url" does not exist`` + This means that the ``paster initdb`` step has not been run successfully. Refer to the installation instructions for this extension. Authorization @@ -72,17 +79,30 @@ Once you have set up your credentials.json file you can generate an oauth token file by using the following command, which will store your oauth token in a file called token.dat once you have finished -giving permission in the browser. +giving permission in the browser:: $ paster getauthtoken --config=../ckan/development.ini + +Now ensure you reference the correct path to your token.dat in your CKAN config file (e.g. development.ini):: + + googleanalytics.token.filepath = ~/pyenv/token.dat Tutorial -------- -Download some GA data and store it in CKAN's db. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file) and specifying the name of your auth file (token.dat by default) from the previous step:: +Download some GA data and store it in CKAN's database. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file) and specifying the name of your auth file (token.dat by default) from the previous step:: - $ paster loadanalytics token.dat latest --config=../ckan/development.ini + $ paster loadanalytics latest --config=../ckan/development.ini + +The value after the token file is how much data you want to retrieve, this can be + +* **all** - data for all time (since 2010) + +* **latest** - (default) just the 'latest' data + +* **YYYY-MM-DD** - just data for all time periods going back to (and including) this date + Software Licence --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -1,7 +1,13 @@ import logging +import datetime +import os + +from pylons import config from ckan.lib.cli import CkanCommand -# No other CKAN imports allowed until _load_config is run, or logging is disabled +# No other CKAN imports allowed until _load_config is run, +# or logging is disabled + class InitDB(CkanCommand): """Initialise the extension's database tables @@ -17,7 +23,7 @@ import ckan.model as model model.Session.remove() model.Session.configure(bind=model.meta.engine) - log = logging.getLogger('ckanext.ga-report') + log = logging.getLogger('ckanext.ga_report') import ga_model ga_model.init_tables() @@ -26,6 +32,12 @@ class GetAuthToken(CkanCommand): """ Get's the Google auth token + + Usage: paster getauthtoken + + Where is the file name containing the details + for the service (obtained from https://code.google.com/apis/console). + By default this is set to credentials.json """ summary = __doc__.split('\n')[0] usage = __doc__ @@ -33,52 +45,108 @@ min_args = 0 def command(self): - from ga_auth import initialize_service - initialize_service('token.dat', - self.args[0] if self.args - else 'credentials.json') + """ + In this case we don't want a valid service, but rather just to + force the user through the auth flow. We allow this to complete to + act as a form of verification instead of just getting the token and + assuming it is correct. + """ + from ga_auth import init_service + init_service('token.dat', + self.args[0] if self.args + else 'credentials.json') + +class FixTimePeriods(CkanCommand): + """ + Fixes the 'All' records for GA_Urls + + It is possible that older urls that haven't recently been visited + do not have All records. This command will traverse through those + records and generate valid All records for them. + """ + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = 0 + min_args = 0 + + def __init__(self, name): + super(FixTimePeriods, self).__init__(name) + + def command(self): + import ckan.model as model + from ga_model import post_update_url_stats + self._load_config() + model.Session.remove() + model.Session.configure(bind=model.meta.engine) + + log = logging.getLogger('ckanext.ga_report') + + log.info("Updating 'All' records for old URLs") + post_update_url_stats() + log.info("Processing complete") + + class LoadAnalytics(CkanCommand): """Get data from Google Analytics API and save it in the ga_model - Usage: paster loadanalytics + Usage: paster loadanalytics - Where is the name of the auth token file from - the getauthtoken step. - - And where is: + Where is: all - data for all time latest - (default) just the 'latest' data - YYYY-MM-DD - just data for all time periods going - back to (and including) this date + YYYY-MM - just data for the specific month """ summary = __doc__.split('\n')[0] usage = __doc__ - max_args = 2 - min_args = 1 + max_args = 1 + min_args = 0 + + def __init__(self, name): + super(LoadAnalytics, self).__init__(name) + self.parser.add_option('-d', '--delete-first', + action='store_true', + default=False, + dest='delete_first', + help='Delete data for the period first') + self.parser.add_option('-s', '--skip_url_stats', + action='store_true', + default=False, + dest='skip_url_stats', + help='Skip the download of URL data - just do site-wide stats') def command(self): self._load_config() - from ga_auth import initialize_service - try: - svc = initialize_service(self.args[0], None) - except TypeError: - print 'Have you correctly run the getauthtoken task and specified the correct file here' + from download_analytics import DownloadAnalytics + from ga_auth import (init_service, get_profile_id) + + ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', '')) + if not ga_token_filepath: + print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ + 'Google Analytics token file under key: googleanalytics.token.filepath' return - from download_analytics import DownloadAnalytics - from ga_auth import get_profile_id - downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) + try: + svc = init_service(ga_token_filepath, None) + except TypeError: + print ('Have you correctly run the getauthtoken task and ' + 'specified the correct token file in the CKAN config under ' + '"googleanalytics.token.filepath"?') + return - time_period = self.args[1] if self.args and len(self.args) > 1 else 'latest' + downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc), + delete_first=self.options.delete_first, + skip_url_stats=self.options.skip_url_stats) + + time_period = self.args[0] if self.args else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': downloader.latest() else: - since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d') - downloader.since_date(since_date) + # The month to use + for_date = datetime.datetime.strptime(time_period, '%Y-%m') + downloader.specific_month(for_date) - --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,10 +1,528 @@ +import re +import csv +import sys +import json import logging -from ckan.lib.base import BaseController, c, render -import report_model +import operator +import collections +from ckan.lib.base import (BaseController, c, g, render, request, response, abort) + +import sqlalchemy +from sqlalchemy import func, cast, Integer +import ckan.model as model +from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher log = logging.getLogger('ckanext.ga-report') +DOWNLOADS_AVAILABLE_FROM = '2012-12' + +def _get_month_name(strdate): + import calendar + from time import strptime + d = strptime(strdate, '%Y-%m') + return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) + +def _get_unix_epoch(strdate): + from time import strptime,mktime + d = strptime(strdate, '%Y-%m') + return int(mktime(d)) + +def _month_details(cls, stat_key=None): + ''' + Returns a list of all the periods for which we have data, unfortunately + knows too much about the type of the cls being passed as GA_Url has a + more complex query + + This may need extending if we add a period_name to the stats + ''' + months = [] + day = None + + q = model.Session.query(cls.period_name,cls.period_complete_day)\ + .filter(cls.period_name!='All').distinct(cls.period_name) + if stat_key: + q= q.filter(cls.stat_name==stat_key) + + vals = q.order_by("period_name desc").all() + + if vals and vals[0][1]: + day = int(vals[0][1]) + ordinal = 'th' if 11 <= day <= 13 \ + else {1:'st',2:'nd',3:'rd'}.get(day % 10, 'th') + day = "{day}{ordinal}".format(day=day, ordinal=ordinal) + + for m in vals: + months.append( (m[0], _get_month_name(m[0]))) + + return months, day + + class GaReport(BaseController): + + def csv(self, month): + import csv + + q = model.Session.query(GA_Stat).filter(GA_Stat.stat_name!='Downloads') + if month != 'all': + q = q.filter(GA_Stat.period_name==month) + entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() + + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=stats_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Period", "Statistic", "Key", "Value"]) + + for entry in entries: + writer.writerow([entry.period_name.encode('utf-8'), + entry.stat_name.encode('utf-8'), + entry.key.encode('utf-8'), + entry.value.encode('utf-8')]) + + def index(self): - return render('index.html') - + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months, c.day = _month_details(GA_Stat) + + # Work out which month to show, based on query params of the first item + c.month_desc = 'all months' + c.month = request.params.get('month', '') + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals') + if c.month: + q = q.filter(GA_Stat.period_name==c.month) + entries = q.order_by('ga_stat.key').all() + + def clean_key(key, val): + if key in ['Average time on site', 'Pages per visit', 'New visits', 'Bounce rate (home page)']: + val = "%.2f" % round(float(val), 2) + if key == 'Average time on site': + mins, secs = divmod(float(val), 60) + hours, mins = divmod(mins, 60) + val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) + if key in ['New visits','Bounce rate (home page)']: + val = "%s%%" % val + if key in ['Total page views', 'Total visits']: + val = int(val) + + return key, val + + # Query historic values for sparkline rendering + sparkline_query = model.Session.query(GA_Stat)\ + .filter(GA_Stat.stat_name=='Totals')\ + .order_by(GA_Stat.period_name) + sparkline_data = {} + for x in sparkline_query: + sparkline_data[x.key] = sparkline_data.get(x.key,[]) + key, val = clean_key(x.key,float(x.value)) + tooltip = '%s: %s' % (_get_month_name(x.period_name), val) + sparkline_data[x.key].append( (tooltip,x.value) ) + # Trim the latest month, as it looks like a huge dropoff + for key in sparkline_data: + sparkline_data[key] = sparkline_data[key][:-1] + + c.global_totals = [] + if c.month: + for e in entries: + key, val = clean_key(e.key, e.value) + sparkline = sparkline_data[e.key] + c.global_totals.append((key, val, sparkline)) + else: + d = collections.defaultdict(list) + for e in entries: + d[e.key].append(float(e.value)) + for k, v in d.iteritems(): + if k in ['Total page views', 'Total visits']: + v = sum(v) + else: + v = float(sum(v))/float(len(v)) + sparkline = sparkline_data[k] + key, val = clean_key(k,v) + + c.global_totals.append((key, val, sparkline)) + # Sort the global totals into a more pleasant order + def sort_func(x): + key = x[0] + total_order = ['Total page views','Total visits','Pages per visit'] + if key in total_order: + return total_order.index(key) + return 999 + c.global_totals = sorted(c.global_totals, key=sort_func) + + keys = { + 'Browser versions': 'browser_versions', + 'Browsers': 'browsers', + 'Operating Systems versions': 'os_versions', + 'Operating Systems': 'os', + 'Social sources': 'social_networks', + 'Languages': 'languages', + 'Country': 'country' + } + + def shorten_name(name, length=60): + return (name[:length] + '..') if len(name) > 60 else name + + def fill_out_url(url): + import urlparse + return urlparse.urljoin(g.site_url, url) + + c.social_referrer_totals, c.social_referrers = [], [] + q = model.Session.query(GA_ReferralStat) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('ga_referrer.count::int desc') + for entry in q.all(): + c.social_referrers.append((shorten_name(entry.url), fill_out_url(entry.url), + entry.source,entry.count)) + + q = model.Session.query(GA_ReferralStat.url, + func.sum(GA_ReferralStat.count).label('count')) + q = q.filter(GA_ReferralStat.period_name==c.month) if c.month else q + q = q.order_by('count desc').group_by(GA_ReferralStat.url) + for entry in q.all(): + c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'', + entry[1])) + + for k, v in keys.iteritems(): + q = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k).\ + order_by(GA_Stat.period_name) + # Run the query on all months to gather graph data + graph = {} + for stat in q: + graph[ stat.key ] = graph.get(stat.key,{ + 'name':stat.key, + 'data': [] + }) + graph[ stat.key ]['data'].append({ + 'x':_get_unix_epoch(stat.period_name), + 'y':float(stat.value) + }) + setattr(c, v+'_graph', json.dumps( _to_rickshaw(graph.values(),percentageMode=True) )) + + # Buffer the tabular data + if c.month: + entries = [] + q = q.filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc') + + d = collections.defaultdict(int) + for e in q.all(): + d[e.key] += int(e.value) + entries = [] + for key, val in d.iteritems(): + entries.append((key,val,)) + entries = sorted(entries, key=operator.itemgetter(1), reverse=True) + + # Get the total for each set of values and then set the value as + # a percentage of the total + if k == 'Social sources': + total = sum([x for n,x,graph in c.global_totals if n == 'Total visits']) + else: + total = sum([num for _,num in entries]) + setattr(c, v, [(k,_percent(v,total)) for k,v in entries ]) + + return render('ga_report/site/index.html') + + +class GaDatasetReport(BaseController): + """ + Displays the pageview and visit count for datasets + with options to filter by publisher and time period. + """ + def publisher_csv(self, month): + ''' + Returns a CSV of each publisher with the total number of dataset + views & visits. + ''' + c.month = month if not month == 'all' else '' + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,)) + + writer = csv.writer(response) + writer.writerow(["Publisher Title", "Publisher Name", "Views", "Visits", "Period Name"]) + + top_publishers, top_publishers_graph = _get_top_publishers(None) + + for publisher,view,visit in top_publishers: + writer.writerow([publisher.title.encode('utf-8'), + publisher.name.encode('utf-8'), + view, + visit, + month]) + + def dataset_csv(self, id='all', month='all'): + ''' + Returns a CSV with the number of views & visits for each dataset. + + :param id: A Publisher ID or None if you want for all + :param month: The time period, or 'all' + ''' + c.month = month if not month == 'all' else '' + if id != 'all': + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + + packages = self._get_packages(c.publisher) + response.headers['Content-Type'] = "text/csv; charset=utf-8" + response.headers['Content-Disposition'] = \ + str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,)) + + writer = csv.writer(response) + writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"]) + + for package,view,visit,downloads in packages: + writer.writerow([package.title.encode('utf-8'), + package.name.encode('utf-8'), + view, + visit, + downloads, + month]) + + def publishers(self): + '''A list of publishers and the number of views/visits for each''' + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months, c.day = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', '') + c.month_desc = 'all months' + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + c.top_publishers, graph_data = _get_top_publishers() + c.top_publishers_graph = json.dumps( _to_rickshaw(graph_data.values()) ) + + return render('ga_report/publisher/index.html') + + def _get_packages(self, publisher=None, count=-1): + '''Returns the datasets in order of views''' + have_download_data = True + month = c.month or 'All' + if month != 'All': + have_download_data = month >= DOWNLOADS_AVAILABLE_FROM + + q = model.Session.query(GA_Url,model.Package)\ + .filter(model.Package.name==GA_Url.package_id)\ + .filter(GA_Url.url.like('/dataset/%')) + if publisher: + q = q.filter(GA_Url.department_id==publisher.name) + q = q.filter(GA_Url.period_name==month) + q = q.order_by('ga_url.pageviews::int desc') + top_packages = [] + if count == -1: + entries = q.all() + else: + entries = q.limit(count) + + for entry,package in entries: + if package: + # Downloads .... + if have_download_data: + dls = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Downloads').\ + filter(GA_Stat.key==package.name) + if month != 'All': # Fetch everything unless the month is specific + dls = dls.filter(GA_Stat.period_name==month) + downloads = 0 + for x in dls: + downloads += int(x.value) + else: + downloads = 'No data' + top_packages.append((package, entry.pageviews, entry.visits, downloads)) + else: + log.warning('Could not find package associated package') + + return top_packages + + def read(self): + ''' + Lists the most popular datasets across all publishers + ''' + return self.read_publisher(None) + + def read_publisher(self, id): + ''' + Lists the most popular datasets for a publisher (or across all publishers) + ''' + count = 20 + + c.publishers = _get_publishers() + + id = request.params.get('publisher', id) + if id and id != 'all': + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') + c.publisher_name = c.publisher.name + c.top_packages = [] # package, dataset_views in c.top_packages + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months, c.day = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all months' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + month = c.month or 'All' + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher_name) + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + + c.top_packages = self._get_packages(c.publisher, 20) + + # Graph query + top_package_names = [ x[0].name for x in c.top_packages ] + graph_query = model.Session.query(GA_Url,model.Package)\ + .filter(model.Package.name==GA_Url.package_id)\ + .filter(GA_Url.url.like('/dataset/%'))\ + .filter(GA_Url.package_id.in_(top_package_names)) + graph_data = {} + for entry,package in graph_query: + if not package: continue + if entry.period_name=='All': continue + graph_data[package.id] = graph_data.get(package.id,{ + 'name':package.title, + 'data':[] + }) + graph_data[package.id]['data'].append({ + 'x':_get_unix_epoch(entry.period_name), + 'y':int(entry.pageviews), + }) + + c.graph_data = json.dumps( _to_rickshaw(graph_data.values()) ) + + return render('ga_report/publisher/read.html') + +def _to_rickshaw(data, percentageMode=False): + if data==[]: + return data + # Create a consistent x-axis + num_points = [ len(package['data']) for package in data ] + ideal_index = num_points.index( max(num_points) ) + x_axis = [ point['x'] for point in data[ideal_index]['data'] ] + for package in data: + xs = [ point['x'] for point in package['data'] ] + assert set(xs).issubset( set(x_axis) ), (xs, x_axis) + # Zero pad any missing values + for x in set(x_axis).difference(set(xs)): + package['data'].append( {'x':x, 'y':0} ) + assert len(package['data'])==len(x_axis), (len(package['data']),len(x_axis),package['data'],x_axis,set(x_axis).difference(set(xs))) + if percentageMode: + # Transform data into percentage stacks + totals = {} + for x in x_axis: + for package in data: + for point in package['data']: + totals[ point['x'] ] = totals.get(point['x'],0) + point['y'] + # Roll insignificant series into a catch-all + THRESHOLD = 0.01 + significant_series = [] + for package in data: + for point in package['data']: + fraction = float(point['y']) / totals[point['x']] + if fraction>THRESHOLD and not (package in significant_series): + significant_series.append(package) + temp = {} + for package in data: + if package in significant_series: continue + for point in package['data']: + temp[point['x']] = temp.get(point['x'],0) + point['y'] + catch_all = { 'name':'Other','data': [ {'x':x,'y':y} for x,y in temp.items() ] } + # Roll insignificant series into one + data = significant_series + data.append(catch_all) + # Turn each point into a percentage + for package in data: + for point in package['data']: + point['y'] = (point['y']*100) / totals[point['x']] + # Sort the points + for package in data: + package['data'] = sorted( package['data'], key=lambda x:x['x'] ) + # Strip the latest month's incomplete analytics + package['data'] = package['data'][:-1] + return data + + +def _get_top_publishers(limit=20): + ''' + Returns a list of the top 20 publishers by dataset visits. + (The number to show can be varied with 'limit') + ''' + month = c.month or 'All' + connection = model.Session.connection() + q = """ + select department_id, sum(pageviews::int) views, sum(visits::int) visits + from ga_url + where department_id <> '' + and package_id <> '' + and url like '/dataset/%%' + and period_name=%s + group by department_id order by views desc + """ + if limit: + q = q + " limit %s;" % (limit) + + top_publishers = [] + res = connection.execute(q, month) + department_ids = [] + for row in res: + g = model.Group.get(row[0]) + if g: + department_ids.append(row[0]) + top_publishers.append((g, row[1], row[2])) + + graph = {} + if limit is not None: + # Query for a history graph of these publishers + q = model.Session.query( + GA_Url.department_id, + GA_Url.period_name, + func.sum(cast(GA_Url.pageviews,sqlalchemy.types.INT)))\ + .filter( GA_Url.department_id.in_(department_ids) )\ + .filter( GA_Url.period_name!='All' )\ + .filter( GA_Url.url.like('/dataset/%') )\ + .filter( GA_Url.package_id!='' )\ + .group_by( GA_Url.department_id, GA_Url.period_name ) + for dept_id,period_name,views in q: + graph[dept_id] = graph.get( dept_id, { + 'name' : model.Group.get(dept_id).title, + 'data' : [] + }) + graph[dept_id]['data'].append({ + 'x': _get_unix_epoch(period_name), + 'y': views + }) + return top_publishers, graph + + +def _get_publishers(): + ''' + Returns a list of all publishers. Each item is a tuple: + (name, title) + ''' + publishers = [] + for pub in model.Session.query(model.Group).\ + filter(model.Group.type=='publisher').\ + filter(model.Group.state=='active').\ + order_by(model.Group.name): + publishers.append((pub.name, pub.title)) + return publishers + +def _percent(num, total): + p = 100 * float(num)/float(total) + return "%.2f%%" % round(p, 2) + --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -1,8 +1,9 @@ +import os import logging import datetime - +import collections from pylons import config - +from ga_model import _normalize_url import ga_model #from ga_client import GA @@ -10,18 +11,37 @@ log = logging.getLogger('ckanext.ga-report') FORMAT_MONTH = '%Y-%m' +MIN_VIEWS = 50 +MIN_VISITS = 20 +MIN_DOWNLOADS = 10 class DownloadAnalytics(object): '''Downloads and stores analytics info''' - def __init__(self, service=None, profile_id=None): + def __init__(self, service=None, profile_id=None, delete_first=False, + skip_url_stats=False): self.period = config['ga-report.period'] self.service = service self.profile_id = profile_id - - - def all_(self): - self.since_date(datetime.datetime(2010, 1, 1)) + self.delete_first = delete_first + self.skip_url_stats = skip_url_stats + + def specific_month(self, date): + import calendar + + first_of_this_month = datetime.datetime(date.year, date.month, 1) + _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month)) + last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month) + # if this is the latest month, note that it is only up until today + now = datetime.datetime.now() + if now.year == date.year and now.month == date.month: + last_day_of_month = now.day + last_of_this_month = now + periods = ((date.strftime(FORMAT_MONTH), + last_day_of_month, + first_of_this_month, last_of_this_month),) + self.download_and_store(periods) + def latest(self): if self.period == 'monthly': @@ -36,13 +56,13 @@ self.download_and_store(periods) - def since_date(self, since_date): + def for_date(self, for_date): assert isinstance(since_date, datetime.datetime) periods = [] # (period_name, period_complete_day, start_date, end_date) if self.period == 'monthly': first_of_the_months_until_now = [] - year = since_date.year - month = since_date.month + year = for_date.year + month = for_date.month now = datetime.datetime.now() first_of_this_month = datetime.datetime(now.year, now.month, 1) while True: @@ -80,27 +100,81 @@ def download_and_store(self, periods): for period_name, period_complete_day, start_date, end_date in periods: - log.info('Downloading Analytics for period "%s" (%s - %s)', + log.info('Period "%s" (%s - %s)', self.get_full_period_name(period_name, period_complete_day), - start_date.strftime('%Y %m %d'), - end_date.strftime('%Y %m %d')) - data = self.download(start_date, end_date) - log.info('Storing Analytics for period "%s"', - self.get_full_period_name(period_name, period_complete_day)) - self.store(period_name, period_complete_day, data) - - - def download(self, start_date, end_date): + start_date.strftime('%Y-%m-%d'), + end_date.strftime('%Y-%m-%d')) + + if self.delete_first: + log.info('Deleting existing Analytics for this period "%s"', + period_name) + ga_model.delete(period_name) + + if not self.skip_url_stats: + # Clean out old url data before storing the new + ga_model.pre_update_url_stats(period_name) + + accountName = config.get('googleanalytics.account') + + log.info('Downloading analytics for dataset views') + data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName) + + log.info('Storing dataset views (%i rows)', len(data.get('url'))) + self.store(period_name, period_complete_day, data, ) + + log.info('Downloading analytics for publisher views') + data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName) + + log.info('Storing publisher views (%i rows)', len(data.get('url'))) + self.store(period_name, period_complete_day, data,) + + # Make sure the All records are correct. + ga_model.post_update_url_stats() + + log.info('Associating datasets with their publisher') + ga_model.update_publisher_stats(period_name) # about 30 seconds. + + + log.info('Downloading and storing analytics for site-wide stats') + self.sitewide_stats( period_name, period_complete_day ) + + log.info('Downloading and storing analytics for social networks') + self.update_social_info(period_name, start_date, end_date) + + + def update_social_info(self, period_name, start_date, end_date): + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + query = 'ga:hasSocialSourceReferral=~Yes$' + metrics = 'ga:entrances' + sort = '-ga:entrances' + + # Supported query params at + # https://developers.google.com/analytics/devguides/reporting/core/v3/reference + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters=query, + start_date=start_date, + metrics=metrics, + sort=sort, + dimensions="ga:landingPagePath,ga:socialNetwork", + max_results=10000, + end_date=end_date).execute() + data = collections.defaultdict(list) + rows = results.get('rows',[]) + for row in rows: + url = _normalize_url('http:/' + row[0]) + data[url].append( (row[1], int(row[2]),) ) + ga_model.update_social(period_name, data) + + + def download(self, start_date, end_date, path=None): '''Get data from GA for a given time period''' start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') - # url - #query = 'ga:pagePath=~^%s,ga:pagePath=~^%s' % \ - # (PACKAGE_URL, self.resource_url_tag) - query = 'ga:pagePath=~^/dataset/' - #query = 'ga:pagePath=~^/User/' - metrics = 'ga:uniquePageviews' - sort = '-ga:uniquePageviews' + query = 'ga:pagePath=%s$' % path + metrics = 'ga:pageviews, ga:visits' + sort = '-ga:pageviews' # Supported query params at # https://developers.google.com/analytics/devguides/reporting/core/v3/reference @@ -110,35 +184,317 @@ start_date=start_date, metrics=metrics, sort=sort, - end_date=end_date).execute() - self.print_results(results) - -# for entry in GA.ga_query(query_filter=query, -# from_date=start_date, -# metrics=metrics, -# sort=sort, -# to_date=end_date): -# print entry, type(entry) -# import pdb; pdb.set_trace() -# for dim in entry.dimension: -# if dim.name == "ga:pagePath": -# package = dim.value -# count = entry.get_metric( -# 'ga:uniquePageviews').value or 0 -# packages[package] = int(count) - return [] - - def print_results(self, results): - import pprint - pprint.pprint(results) - if results: - print 'Profile: %s' % results.get('profileInfo').get('profileName') - print 'Total results: %s' % results.get('totalResults') - print 'Total Visits: %s' % results.get('rows', [[-1]])[0][0] - else: - print 'No results found' + dimensions="ga:pagePath", + max_results=10000, + end_date=end_date).execute() + + packages = [] + log.info("There are %d results" % results['totalResults']) + for entry in results.get('rows'): + (loc,pageviews,visits) = entry + url = _normalize_url('http:/' + loc) # strips off domain e.g. www.data.gov.uk or data.gov.uk + + if not url.startswith('/dataset/') and not url.startswith('/publisher/'): + # filter out strays like: + # /data/user/login?came_from=http://data.gov.uk/dataset/os-code-point-open + # /403.html?page=/about&from=http://data.gov.uk/publisher/planning-inspectorate + continue + packages.append( (url, pageviews, visits,) ) # Temporary hack + return dict(url=packages) def store(self, period_name, period_complete_day, data): if 'url' in data: ga_model.update_url_stats(period_name, period_complete_day, data['url']) + def sitewide_stats(self, period_name, period_complete_day): + import calendar + year, month = period_name.split('-') + _, last_day_of_month = calendar.monthrange(int(year), int(month)) + + start_date = '%s-01' % period_name + end_date = '%s-%s' % (period_name, last_day_of_month) + funcs = ['_totals_stats', '_social_stats', '_os_stats', + '_locale_stats', '_browser_stats', '_mobile_stats', '_download_stats'] + for f in funcs: + log.info('Downloading analytics for %s' % f.split('_')[1]) + getattr(self, f)(start_date, end_date, period_name, period_complete_day) + + def _get_results(result_data, f): + data = {} + for result in result_data: + key = f(result) + data[key] = data.get(key,0) + result[1] + return data + + def _totals_stats(self, start_date, end_date, period_name, period_complete_day): + """ Fetches distinct totals, total pageviews etc """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}, + period_complete_day) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visits', + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = { + 'Pages per visit': result_data[0][0], + 'Average time on site': result_data[0][1], + 'New visits': result_data[0][2], + 'Total visits': result_data[0][3], + } + ga_model.update_sitewide_stats(period_name, "Totals", data, period_complete_day) + + # Bounces from / or another configurable page. + path = '/%s%s' % (config.get('googleanalytics.account'), + config.get('ga-report.bounce_url', '/')) + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters='ga:pagePath==%s' % (path,), + start_date=start_date, + metrics='ga:visitBounceRate', + dimensions='ga:pagePath', + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + if not result_data or len(result_data) != 1: + log.error('Could not pinpoint the bounces for path: %s. Got results: %r', + path, result_data) + return + results = result_data[0] + bounces = float(results[1]) + # visitBounceRate is already a % + log.info('Google reports visitBounceRate as %s', bounces) + ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': float(bounces)}, + period_complete_day) + + + def _locale_stats(self, start_date, end_date, period_name, period_complete_day): + """ Fetches stats about language and country """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + dimensions="ga:language,ga:country", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Languages", data, period_complete_day) + + data = {} + for result in result_data: + data[result[1]] = data.get(result[1], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Country", data, period_complete_day) + + + def _download_stats(self, start_date, end_date, period_name, period_complete_day): + """ Fetches stats about data downloads """ + import ckan.model as model + + data = {} + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + filters='ga:eventAction==download', + metrics='ga:totalEvents', + sort='-ga:totalEvents', + dimensions="ga:eventLabel", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + if not result_data: + # We may not have data for this time period, so we need to bail + # early. + log.info("There is no download data for this time period") + return + + def process_result_data(result_data, cached=False): + progress_total = len(result_data) + progress_count = 0 + resources_not_matched = [] + for result in result_data: + progress_count += 1 + if progress_count % 100 == 0: + log.debug('.. %d/%d done so far', progress_count, progress_total) + + url = result[0].strip() + + # Get package id associated with the resource that has this URL. + q = model.Session.query(model.Resource) + if cached: + r = q.filter(model.Resource.cache_url.like("%s%%" % url)).first() + else: + r = q.filter(model.Resource.url.like("%s%%" % url)).first() + + package_name = r.resource_group.package.name if r else "" + if package_name: + data[package_name] = data.get(package_name, 0) + int(result[1]) + else: + resources_not_matched.append(url) + continue + if resources_not_matched: + log.debug('Could not match %i or %i resource URLs to datasets. e.g. %r', + len(resources_not_matched), progress_total, resources_not_matched[:3]) + + log.info('Associating downloads of resource URLs with their respective datasets') + process_result_data(results.get('rows')) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + filters='ga:eventAction==download-cache', + metrics='ga:totalEvents', + sort='-ga:totalEvents', + dimensions="ga:eventLabel", + max_results=10000, + end_date=end_date).execute() + log.info('Associating downloads of cache resource URLs with their respective datasets') + process_result_data(results.get('rows'), cached=False) + + self._filter_out_long_tail(data, MIN_DOWNLOADS) + ga_model.update_sitewide_stats(period_name, "Downloads", data, period_complete_day) + + def _social_stats(self, start_date, end_date, period_name, period_complete_day): + """ Finds out which social sites people are referred from """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + dimensions="ga:socialNetwork,ga:referralPath", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + if not result[0] == '(not set)': + data[result[0]] = data.get(result[0], 0) + int(result[2]) + self._filter_out_long_tail(data, 3) + ga_model.update_sitewide_stats(period_name, "Social sources", data, period_complete_day) + + + def _os_stats(self, start_date, end_date, period_name, period_complete_day): + """ Operating system stats """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + dimensions="ga:operatingSystem,ga:operatingSystemVersion", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Operating Systems", data, period_complete_day) + + data = {} + for result in result_data: + if int(result[2]) >= MIN_VIEWS: + key = "%s %s" % (result[0],result[1]) + data[key] = result[2] + ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data, period_complete_day) + + + def _browser_stats(self, start_date, end_date, period_name, period_complete_day): + """ Information about browsers and browser versions """ + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + dimensions="ga:browser,ga:browserVersion", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + # e.g. [u'Firefox', u'19.0', u'20'] + + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Browsers", data, period_complete_day) + + data = {} + for result in result_data: + key = "%s %s" % (result[0], self._filter_browser_version(result[0], result[1])) + data[key] = data.get(key, 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Browser versions", data, period_complete_day) + + @classmethod + def _filter_browser_version(cls, browser, version_str): + ''' + Simplifies a browser version string if it is detailed. + i.e. groups together Firefox 3.5.1 and 3.5.2 to be just 3. + This is helpful when viewing stats and good to protect privacy. + ''' + ver = version_str + parts = ver.split('.') + if len(parts) > 1: + if parts[1][0] == '0': + ver = parts[0] + else: + ver = "%s" % (parts[0]) + # Special case complex version nums + if browser in ['Safari', 'Android Browser']: + ver = parts[0] + if len(ver) > 2: + num_hidden_digits = len(ver) - 2 + ver = ver[0] + ver[1] + 'X' * num_hidden_digits + return ver + + def _mobile_stats(self, start_date, end_date, period_name, period_complete_day): + """ Info about mobile devices """ + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + metrics='ga:pageviews', + sort='-ga:pageviews', + dimensions="ga:mobileDeviceBranding, ga:mobileDeviceInfo", + max_results=10000, + end_date=end_date).execute() + + result_data = results.get('rows') + data = {} + for result in result_data: + data[result[0]] = data.get(result[0], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Mobile brands", data, period_complete_day) + + data = {} + for result in result_data: + data[result[1]] = data.get(result[1], 0) + int(result[2]) + self._filter_out_long_tail(data, MIN_VIEWS) + ga_model.update_sitewide_stats(period_name, "Mobile devices", data, period_complete_day) + + @classmethod + def _filter_out_long_tail(cls, data, threshold=10): + ''' + Given data which is a frequency distribution, filter out + results which are below a threshold count. This is good to protect + privacy. + ''' + for key, value in data.items(): + if value < threshold: + del data[key] + --- a/ckanext/ga_report/ga_auth.py +++ b/ckanext/ga_report/ga_auth.py @@ -1,3 +1,4 @@ +import os import httplib2 from apiclient.discovery import build from oauth2client.client import flow_from_clientsecrets @@ -7,8 +8,12 @@ from pylons import config -def _prepare_credentials( token_filename, credentials_filename ): - storage = Storage( token_filename ) +def _prepare_credentials(token_filename, credentials_filename): + """ + Either returns the user's oauth credentials or uses the credentials + file to generate a token (by forcing the user to login in the browser) + """ + storage = Storage(token_filename) credentials = storage.get() if credentials is None or credentials.invalid: @@ -19,7 +24,13 @@ return credentials -def initialize_service( token_file, credentials_file ): + +def init_service(token_file, credentials_file): + """ + Given a file containing the user's oauth token (and another with + credentials in case we need to generate the token) will return a + service object representing the analytics API. + """ http = httplib2.Http() credentials = _prepare_credentials(token_file, credentials_file) @@ -27,19 +38,37 @@ return build('analytics', 'v3', http=http) + def get_profile_id(service): - # Get a list of all Google Analytics accounts for this user + """ + Get the profile ID for this user and the service specified by the + 'googleanalytics.id' configuration option. This function iterates + over all of the accounts available to the user who invoked the + service to find one where the account name matches (in case the + user has several). + """ accounts = service.management().accounts().list().execute() - if accounts.get('items'): - firstAccountId = accounts.get('items')[0].get('id') - webPropertyId = config.get('googleanalytics.id') - profiles = service.management().profiles().list( - accountId=firstAccountId, - webPropertyId=webPropertyId).execute() + if not accounts.get('items'): + return None - if profiles.get('items'): - # return the first Profile ID - return profiles.get('items')[0].get('id') + accountName = config.get('googleanalytics.account') + if not accountName: + raise Exception('googleanalytics.account needs to be configured') + webPropertyId = config.get('googleanalytics.id') + if not webPropertyId: + raise Exception('googleanalytics.id needs to be configured') + for acc in accounts.get('items'): + if acc.get('name') == accountName: + accountId = acc.get('id') + + webproperties = service.management().webproperties().list(accountId=accountId).execute() + + profiles = service.management().profiles().list( + accountId=accountId, webPropertyId=webPropertyId).execute() + + if profiles.get('items'): + return profiles.get('items')[0].get('id') return None + --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -1,32 +1,103 @@ import re import uuid -from sqlalchemy import Table, Column, MetaData +from sqlalchemy import Table, Column, MetaData, ForeignKey from sqlalchemy import types -from sqlalchemy.sql import select, text +from sqlalchemy.sql import select +from sqlalchemy.orm import mapper, relation from sqlalchemy import func import ckan.model as model -from ckan.model.types import JsonType from ckan.lib.base import * + +log = __import__('logging').getLogger(__name__) def make_uuid(): return unicode(uuid.uuid4()) +metadata = MetaData() + +class GA_Url(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +url_table = Table('ga_url', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('period_complete_day', types.Integer), + Column('pageviews', types.UnicodeText), + Column('visits', types.UnicodeText), + Column('url', types.UnicodeText), + Column('department_id', types.UnicodeText), + Column('package_id', types.UnicodeText), + ) +mapper(GA_Url, url_table) + + +class GA_Stat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +stat_table = Table('ga_stat', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('period_complete_day', types.UnicodeText), + Column('stat_name', types.UnicodeText), + Column('key', types.UnicodeText), + Column('value', types.UnicodeText), ) +mapper(GA_Stat, stat_table) + + +class GA_Publisher(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +pub_table = Table('ga_publisher', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('publisher_name', types.UnicodeText), + Column('views', types.UnicodeText), + Column('visits', types.UnicodeText), + Column('toplevel', types.Boolean, default=False), + Column('subpublishercount', types.Integer, default=0), + Column('parent', types.UnicodeText), +) +mapper(GA_Publisher, pub_table) + + +class GA_ReferralStat(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +referrer_table = Table('ga_referrer', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('source', types.UnicodeText), + Column('url', types.UnicodeText), + Column('count', types.Integer), + ) +mapper(GA_ReferralStat, referrer_table) + + + def init_tables(): - metadata = MetaData() - package_stats = Table('ga_url', metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('period_name', types.UnicodeText), - Column('period_complete_day', types.Integer), - Column('visits', types.Integer), - Column('group_id', types.String(60)), - Column('next_page', JsonType), - ) metadata.create_all(model.meta.engine) cached_tables = {} + def get_table(name): if name not in cached_tables: @@ -43,10 +114,10 @@ >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices') '/dataset/weekly_fuel_prices' ''' - url = re.sub('https?://(www\.)?data.gov.uk', '', url) - return url - -def _get_department_id_of_url(url): + return '/' + '/'.join(url.split('/')[3:]) + + +def _get_package_and_publisher(url): # e.g. /dataset/fuel_prices # e.g. /dataset/fuel_prices/resource/e63380d4 dataset_match = re.match('/dataset/([^/]+)(/.*)?', url) @@ -56,36 +127,314 @@ if dataset: publisher_groups = dataset.get_groups('publisher') if publisher_groups: - return publisher_groups[0].id - -def update_url_stats(period_name, period_complete_day, url_data): - table = get_table('ga_url') - connection = model.Session.connection() - for url, views, next_page in url_data: - url = _normalize_url(url) - department_id = _get_department_id_of_url(url) - # see if the row for this url & month is in the table already - s = select([func.count(id_col)], - table.c.period_name == period_name, - table.c.url == url) - count = connection.execute(s).fetchone() - if count and count[0]: - # update the row - connection.execute(table.update()\ - .where(table.c.period_name == period_name, - table.c.url == url)\ - .values(period_complete_day=period_complete_day, - views=views, - department_id=department_id, - next_page=next_page)) + return dataset_ref,publisher_groups[0].name + return dataset_ref, None + else: + publisher_match = re.match('/publisher/([^/]+)(/.*)?', url) + if publisher_match: + return None, publisher_match.groups()[0] + return None, None + +def update_sitewide_stats(period_name, stat_name, data, period_complete_day): + for k,v in data.iteritems(): + item = model.Session.query(GA_Stat).\ + filter(GA_Stat.period_name==period_name).\ + filter(GA_Stat.key==k).\ + filter(GA_Stat.stat_name==stat_name).first() + if item: + item.period_name = period_name + item.key = k + item.value = v + item.period_complete_day = period_complete_day + model.Session.add(item) else: # create the row - values = {'period_name': period_name, + values = {'id': make_uuid(), + 'period_name': period_name, + 'period_complete_day': period_complete_day, + 'key': k, + 'value': v, + 'stat_name': stat_name + } + model.Session.add(GA_Stat(**values)) + model.Session.commit() + + +def pre_update_url_stats(period_name): + q = model.Session.query(GA_Url).\ + filter(GA_Url.period_name==period_name) + log.debug("Deleting %d '%s' records" % (q.count(), period_name)) + q.delete() + + q = model.Session.query(GA_Url).\ + filter(GA_Url.period_name == 'All') + log.debug("Deleting %d 'All' records..." % q.count()) + q.delete() + + model.Session.flush() + model.Session.commit() + model.repo.commit_and_remove() + log.debug('...done') + +def post_update_url_stats(): + + """ Check th