From: Ross Jones Date: Tue, 09 Oct 2012 16:16:33 +0000 Subject: Packages up the data from Analytics. X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=f3afebc6bea83666f60ad90418d9fe490f96928a --- Packages up the data from Analytics. - Makes sure we get the right account by requiring a new googleanalytics.account config option which will be used to look up the correct account details. - Changed the model to use the sqlalchemy orm and made sure it does the updates correctly. - Still need to collect further info (such as the group details) --- --- a/README.rst +++ b/README.rst @@ -31,6 +31,7 @@ 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration:: googleanalytics.id = UA-1010101-1 + googleanalytics.account = Account name (i.e. data.gov.uk, see top level item at https://www.google.com/analytics) googleanalytics.username = googleaccount@gmail.com googleanalytics.password = googlepassword ga-report.period = monthly --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -97,8 +97,7 @@ # url #query = 'ga:pagePath=~^%s,ga:pagePath=~^%s' % \ # (PACKAGE_URL, self.resource_url_tag) - query = 'ga:pagePath=~^/dataset/' - #query = 'ga:pagePath=~^/User/' + query = 'ga:pagePath=~/dataset/[a-z0-9-]+$' metrics = 'ga:uniquePageviews' sort = '-ga:uniquePageviews' @@ -110,30 +109,25 @@ start_date=start_date, metrics=metrics, sort=sort, + dimensions="ga:pagePath", end_date=end_date).execute() - self.print_results(results) -# for entry in GA.ga_query(query_filter=query, -# from_date=start_date, -# metrics=metrics, -# sort=sort, -# to_date=end_date): -# print entry, type(entry) -# import pdb; pdb.set_trace() -# for dim in entry.dimension: -# if dim.name == "ga:pagePath": -# package = dim.value -# count = entry.get_metric( -# 'ga:uniquePageviews').value or 0 -# packages[package] = int(count) - return [] + import pprint + pprint.pprint(results) + print 'Total results: %s' % results.get('totalResults') + + packages = [] + for entry in results.get('rows'): + (loc,size,) = entry + packages.append( ('http:/' + loc,size, '',) ) + return dict(url=packages) def print_results(self, results): import pprint pprint.pprint(results) if results: print 'Profile: %s' % results.get('profileInfo').get('profileName') - print 'Total results: %s' % results.get('totalResults') + print 'Total Visits: %s' % results.get('rows', [[-1]])[0][0] else: print 'No results found' --- a/ckanext/ga_report/ga_auth.py +++ b/ckanext/ga_report/ga_auth.py @@ -1,3 +1,4 @@ +import os import httplib2 from apiclient.discovery import build from oauth2client.client import flow_from_clientsecrets @@ -41,15 +42,24 @@ def get_profile_id(service): """ Get the profile ID for this user and the service specified by the - 'googleanalytics.id' configuration option. + 'googleanalytics.id' configuration option. This function iterates + over all of the accounts available to the user who invoked the + service to find one where the account name matches (in case the + user has several). """ accounts = service.management().accounts().list().execute() if not accounts.get('items'): return None - accountId = accounts.get('items')[0].get('id') + accountName = config.get('googleanalytics.account') webPropertyId = config.get('googleanalytics.id') + for acc in accounts.get('items'): + if acc.get('name') == accountName: + accountId = acc.get('id') + + webproperties = service.management().webproperties().list(accountId=accountId).execute() + profiles = service.management().profiles().list( accountId=accountId, webPropertyId=webPropertyId).execute() --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -4,28 +4,38 @@ from sqlalchemy import Table, Column, MetaData from sqlalchemy import types from sqlalchemy.sql import select +from sqlalchemy.orm import mapper from sqlalchemy import func import ckan.model as model -from ckan.model.types import JsonType from ckan.lib.base import * - def make_uuid(): return unicode(uuid.uuid4()) + +class GA_Url(object): + + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + + +metadata = MetaData() +url_table = Table('ga_url', metadata, + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), + Column('period_name', types.UnicodeText), + Column('period_complete_day', types.Integer), + Column('visits', types.Integer), + Column('url', types.UnicodeText), + Column('next_page', types.UnicodeText), + ) +mapper(GA_Url, url_table) + + def init_tables(): - metadata = MetaData() - package_stats = Table('ga_url', metadata, - Column('id', types.UnicodeText, primary_key=True, - default=make_uuid), - Column('period_name', types.UnicodeText), - Column('period_complete_day', types.Integer), - Column('visits', types.Integer), - Column('group_id', types.String(60)), - Column('next_page', JsonType), - ) metadata.create_all(model.meta.engine) @@ -66,32 +76,27 @@ def update_url_stats(period_name, period_complete_day, url_data): table = get_table('ga_url') - connection = model.Session.connection() for url, views, next_page in url_data: url = _normalize_url(url) department_id = _get_department_id_of_url(url) # see if the row for this url & month is in the table already - s = select([func.count(id_col)], - table.c.period_name == period_name, - table.c.url == url) - count = connection.execute(s).fetchone() - if count and count[0]: - # update the row - connection.execute(table.update() - .where(table.c.period_name == period_name, - table.c.url == url) - .values(period_complete_day=period_complete_day, - views=views, - department_id=department_id, - next_page=next_page)) + item = model.Session.query(GA_Url).filter(GA_Url.period_name==period_name).filter(GA_Url.url==url).first() + if item: + item.period_name = period_complete_day=period_complete_day + item.views = views + item.department_id = department_id + item.next_page = next_page + model.Session.add(item) else: # create the row - values = {'period_name': period_name, + values = {'id': make_uuid(), + 'period_name': period_name, 'period_complete_day': period_complete_day, 'url': url, 'views': views, 'department_id': department_id, 'next_page': next_page} - connection.execute(stats.insert(). - values(**values)) + obj = GA_Url(**values) + model.Session.add(obj) + model.Session.commit() --- /dev/null +++ b/ckanext/ga_report/tests/test_auth.py @@ -1,1 +1,41 @@ +import os +from nose.tools import assert_equal +from ckanext.ga_report.ga_auth import (init_service, get_profile_id) +class TestAuth: + + @classmethod + def setup_class(cls): + if not os.path.exists("token.dat") or not os.path.exists("credentials.json"): + print '*' * 60 + print "Tests may not run without first having run the auth process" + print '*' * 60 + + @classmethod + def teardown_class(cls): + pass + + def test_init(self): + try: + res = init_service(None, None) + assert False, "Init service worked without credentials or tokens" + except TypeError: + pass + + def test_init_with_token(self): + res = init_service("token.dat", None) + assert res is not None, "Init service worked without credentials" + + def test_init_with_token_and_credentials(self): + res = init_service("token.dat", "credentials.json") + assert res is not None, "Unable to create service with valid details" + + def test_init_with_redentials(self): + #res = init_service("", "credentials.json") + # Triggers the auth flow via the browser + pass + + def test_get_profile(self): + svc = init_service("token.dat", "credentials.json") + profile = get_profile_id(svc) + assert profile is not None, "Unable to find a profile given configured UA id and user details"