Remove prettyprinting unless DEBUG is set in env and add a new
test for actually making the calls.
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
*.py[co]
+*.py~
+.gitignore
# Packages
*.egg
@@ -13,6 +15,10 @@
develop-eggs
.installed.cfg
+# Private info
+credentials.json
+token.dat
+
# Installer logs
pip-log.txt
--- a/README.md
+++ /dev/null
@@ -1,4 +1,1 @@
-ckanext-ga-report
-=================
-For creating detailed reports of CKAN analytics, sliced by group
--- /dev/null
+++ b/README.rst
@@ -1,1 +1,102 @@
+ckanext-ga-report
+=================
+**Status:** Development
+
+**CKAN Version:** 1.7.1+
+
+
+Overview
+--------
+
+For creating detailed reports of CKAN analytics, including totals per group.
+
+Whereas ckanext-googleanalytics focusses on providing page view stats a recent period and for all time (aimed at end users), ckanext-ga-report is more interested in building regular periodic reports (more for site managers to monitor).
+
+Contents of this extension:
+
+ * Use the CLI tool to download Google Analytics data for each time period into this extension's database tables
+
+ * Users can view the data as web page reports
+
+
+Installation
+------------
+
+1. Activate you CKAN python environment and install this extension's software::
+
+ $ pyenv/bin/activate
+ $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report
+
+2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration::
+
+ googleanalytics.id = UA-1010101-1
+ googleanalytics.account = Account name (i.e. data.gov.uk, see top level item at https://www.google.com/analytics)
+ ga-report.period = monthly
+
+ Note that your credentials will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose.
+
+3. Set up this extension's database tables using a paster command. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file)::
+
+ $ paster initdb --config=../ckan/development.ini
+
+4. Enable the extension in your CKAN config file by adding it to ``ckan.plugins``::
+
+ ckan.plugins = ga-report
+
+
+Authorization
+--------------
+
+Before you can access the data, you need to set up the OAUTH details which you can do by following the `instructions <https://developers.google.com/analytics/resources/tutorials/hello-analytics-api>`_ the outcome of which will be a file called credentials.json which should look like credentials.json.template with the relevant fields completed. These steps are below for convenience:
+
+1. Visit the `Google APIs Console <https://code.google.com/apis/console>`_
+
+2. Sign-in and create a project or use an existing project.
+
+3. In the `Services pane <https://code.google.com/apis/console#:services>`_ , activate Analytics API for your project. If prompted, read and accept the terms of service.
+
+4. Go to the `API Access pane <https://code.google.com/apis/console/#:access>`_
+
+5. Click Create an OAuth 2.0 client ID....
+
+6. Fill out the Branding Information fields and click Next.
+
+7. In Client ID Settings, set Application type to Installed application.
+
+8. Click Create client ID
+
+9. The details you need below are Client ID, Client secret, and Redirect URIs
+
+
+Once you have set up your credentials.json file you can generate an oauth token file by using the
+following command, which will store your oauth token in a file called token.dat once you have finished
+giving permission in the browser::
+
+ $ paster getauthtoken --config=../ckan/development.ini
+
+
+Tutorial
+--------
+
+Download some GA data and store it in CKAN's db. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file) and specifying the name of your auth file (token.dat by default) from the previous step::
+
+ $ paster loadanalytics token.dat latest --config=../ckan/development.ini
+
+The value after the token file is how much data you want to retrieve, this can be
+
+* **all** - data for all time (since 2010)
+
+* **latest** - (default) just the 'latest' data
+
+* **YYYY-MM-DD** - just data for all time periods going back to (and including) this date
+
+
+
+Software Licence
+================
+
+This software is developed by Cabinet Office. It is Crown Copyright and opened up under the Open Government Licence (OGL) (which is compatible with Creative Commons Attibution License).
+
+OGL terms: http://www.nationalarchives.gov.uk/doc/open-government-licence/
+
--- /dev/null
+++ b/ckanext/__init__.py
@@ -1,1 +1,8 @@
+# this is a namespace package
+try:
+ import pkg_resources
+ pkg_resources.declare_namespace(__name__)
+except ImportError:
+ import pkgutil
+ __path__ = pkgutil.extend_path(__path__, __name__)
--- /dev/null
+++ b/ckanext/ga_report/__init__.py
@@ -1,1 +1,8 @@
+# this is a namespace package
+try:
+ import pkg_resources
+ pkg_resources.declare_namespace(__name__)
+except ImportError:
+ import pkgutil
+ __path__ = pkgutil.extend_path(__path__, __name__)
--- /dev/null
+++ b/ckanext/ga_report/command.py
@@ -1,1 +1,101 @@
+import logging
+import datetime
+from ckan.lib.cli import CkanCommand
+# No other CKAN imports allowed until _load_config is run,
+# or logging is disabled
+
+
+class InitDB(CkanCommand):
+ """Initialise the extension's database tables
+ """
+ summary = __doc__.split('\n')[0]
+ usage = __doc__
+ max_args = 0
+ min_args = 0
+
+ def command(self):
+ self._load_config()
+
+ import ckan.model as model
+ model.Session.remove()
+ model.Session.configure(bind=model.meta.engine)
+ log = logging.getLogger('ckanext.ga-report')
+
+ import ga_model
+ ga_model.init_tables()
+ log.info("DB tables are setup")
+
+
+class GetAuthToken(CkanCommand):
+ """ Get's the Google auth token
+
+ Usage: paster getauthtoken <credentials_file>
+
+ Where <credentials_file> is the file name containing the details
+ for the service (obtained from https://code.google.com/apis/console).
+ By default this is set to credentials.json
+ """
+ summary = __doc__.split('\n')[0]
+ usage = __doc__
+ max_args = 0
+ min_args = 0
+
+ def command(self):
+ """
+ In this case we don't want a valid service, but rather just to
+ force the user through the auth flow. We allow this to complete to
+ act as a form of verification instead of just getting the token and
+ assuming it is correct.
+ """
+ from ga_auth import init_service
+ init_service('token.dat',
+ self.args[0] if self.args
+ else 'credentials.json')
+
+
+class LoadAnalytics(CkanCommand):
+ """Get data from Google Analytics API and save it
+ in the ga_model
+
+ Usage: paster loadanalytics <tokenfile> <time-period>
+
+ Where <tokenfile> is the name of the auth token file from
+ the getauthtoken step.
+
+ And where <time-period> is:
+ all - data for all time
+ latest - (default) just the 'latest' data
+ YYYY-MM-DD - just data for all time periods going
+ back to (and including) this date
+ """
+ summary = __doc__.split('\n')[0]
+ usage = __doc__
+ max_args = 2
+ min_args = 1
+
+ def command(self):
+ self._load_config()
+
+ from download_analytics import DownloadAnalytics
+ from ga_auth import (init_service, get_profile_id)
+
+ try:
+ svc = init_service(self.args[0], None)
+ except TypeError:
+ print ('Have you correctly run the getauthtoken task and '
+ 'specified the correct file here')
+ return
+
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+
+ time_period = self.args[1] if self.args and len(self.args) > 1 \
+ else 'latest'
+ if time_period == 'all':
+ downloader.all_()
+ elif time_period == 'latest':
+ downloader.latest()
+ else:
+ since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
+ downloader.since_date(since_date)
+
--- /dev/null
+++ b/ckanext/ga_report/controller.py
@@ -1,1 +1,10 @@
+import logging
+from ckan.lib.base import BaseController, c, render
+import report_model
+log = logging.getLogger('ckanext.ga-report')
+
+class GaReport(BaseController):
+ def index(self):
+ return render('index.html')
+
--- /dev/null
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,1 +1,128 @@
+import os
+import logging
+import datetime
+from pylons import config
+
+import ga_model
+
+#from ga_client import GA
+
+log = logging.getLogger('ckanext.ga-report')
+
+FORMAT_MONTH = '%Y-%m'
+
+class DownloadAnalytics(object):
+ '''Downloads and stores analytics info'''
+
+ def __init__(self, service=None, profile_id=None):
+ self.period = config['ga-report.period']
+ self.service = service
+ self.profile_id = profile_id
+
+
+ def all_(self):
+ self.since_date(datetime.datetime(2010, 1, 1))
+
+ def latest(self):
+ if self.period == 'monthly':
+ # from first of this month to today
+ now = datetime.datetime.now()
+ first_of_this_month = datetime.datetime(now.year, now.month, 1)
+ periods = ((now.strftime(FORMAT_MONTH),
+ now.day,
+ first_of_this_month, now),)
+ else:
+ raise NotImplementedError
+ self.download_and_store(periods)
+
+
+ def since_date(self, since_date):
+ assert isinstance(since_date, datetime.datetime)
+ periods = [] # (period_name, period_complete_day, start_date, end_date)
+ if self.period == 'monthly':
+ first_of_the_months_until_now = []
+ year = since_date.year
+ month = since_date.month
+ now = datetime.datetime.now()
+ first_of_this_month = datetime.datetime(now.year, now.month, 1)
+ while True:
+ first_of_the_month = datetime.datetime(year, month, 1)
+ if first_of_the_month == first_of_this_month:
+ periods.append((now.strftime(FORMAT_MONTH),
+ now.day,
+ first_of_this_month, now))
+ break
+ elif first_of_the_month < first_of_this_month:
+ in_the_next_month = first_of_the_month + datetime.timedelta(40)
+ last_of_the_month = datetime.datetime(in_the_next_month.year,
+ in_the_next_month.month, 1)\
+ - datetime.timedelta(1)
+ periods.append((now.strftime(FORMAT_MONTH), 0,
+ first_of_the_month, last_of_the_month))
+ else:
+ # first_of_the_month has got to the future somehow
+ break
+ month += 1
+ if month > 12:
+ year += 1
+ month = 1
+ else:
+ raise NotImplementedError
+ self.download_and_store(periods)
+
+ @staticmethod
+ def get_full_period_name(period_name, period_complete_day):
+ if period_complete_day:
+ return period_name + ' (up to %ith)' % period_complete_day
+ else:
+ return period_name
+
+
+ def download_and_store(self, periods):
+ for period_name, period_complete_day, start_date, end_date in periods:
+ log.info('Downloading Analytics for period "%s" (%s - %s)',
+ self.get_full_period_name(period_name, period_complete_day),
+ start_date.strftime('%Y %m %d'),
+ end_date.strftime('%Y %m %d'))
+ data = self.download(start_date, end_date)
+ log.info('Storing Analytics for period "%s"',
+ self.get_full_period_name(period_name, period_complete_day))
+ self.store(period_name, period_complete_day, data)
+
+
+ def download(self, start_date, end_date):
+ '''Get data from GA for a given time period'''
+ start_date = start_date.strftime('%Y-%m-%d')
+ end_date = end_date.strftime('%Y-%m-%d')
+ query = 'ga:pagePath=~/dataset/[a-z0-9-]+$'
+ metrics = 'ga:uniquePageviews'
+ sort = '-ga:uniquePageviews'
+
+ # Supported query params at
+ # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
+ results = self.service.data().ga().get(
+ ids='ga:' + self.profile_id,
+ filters=query,
+ start_date=start_date,
+ metrics=metrics,
+ sort=sort,
+ dimensions="ga:pagePath",
+ max_results=10000,
+ end_date=end_date).execute()
+
+ if os.getenv('DEBUG'):
+ import pprint
+ pprint.pprint(results)
+ print 'Total results: %s' % results.get('totalResults')
+
+ packages = []
+ for entry in results.get('rows'):
+ (loc,size,) = entry
+ packages.append( ('http:/' + loc,size, '',) ) # Temporary hack
+ return dict(url=packages)
+
+ def store(self, period_name, period_complete_day, data):
+ if 'url' in data:
+ ga_model.update_url_stats(period_name, period_complete_day, data['url'])
+
--- /dev/null
+++ b/ckanext/ga_report/ga_auth.py
@@ -1,1 +1,70 @@
+import os
+import httplib2
+from apiclient.discovery import build
+from oauth2client.client import flow_from_clientsecrets
+from oauth2client.file import Storage
+from oauth2client.tools import run
+from pylons import config
+
+
+def _prepare_credentials(token_filename, credentials_filename):
+ """
+ Either returns the user's oauth credentials or uses the credentials
+ file to generate a token (by forcing the user to login in the browser)
+ """
+ storage = Storage(token_filename)
+ credentials = storage.get()
+
+ if credentials is None or credentials.invalid:
+ flow = flow_from_clientsecrets(credentials_filename,
+ scope='https://www.googleapis.com/auth/analytics.readonly',
+ message="Can't find the credentials file")
+ credentials = run(flow, storage)
+
+ return credentials
+
+
+def init_service(token_file, credentials_file):
+ """
+ Given a file containing the user's oauth token (and another with
+ credentials in case we need to generate the token) will return a
+ service object representing the analytics API.
+ """
+ http = httplib2.Http()
+
+ credentials = _prepare_credentials(token_file, credentials_file)
+ http = credentials.authorize(http) # authorize the http object
+
+ return build('analytics', 'v3', http=http)
+
+
+def get_profile_id(service):
+ """
+ Get the profile ID for this user and the service specified by the
+ 'googleanalytics.id' configuration option. This function iterates
+ over all of the accounts available to the user who invoked the
+ service to find one where the account name matches (in case the
+ user has several).
+ """
+ accounts = service.management().accounts().list().execute()
+
+ if not accounts.get('items'):
+ return None
+
+ accountName = config.get('googleanalytics.account')
+ webPropertyId = config.get('googleanalytics.id')
+ for acc in accounts.get('items'):
+ if acc.get('name') == accountName:
+ accountId = acc.get('id')
+
+ webproperties = service.management().webproperties().list(accountId=accountId).execute()
+
+ profiles = service.management().profiles().list(
+ accountId=accountId, webPropertyId=webPropertyId).execute()
+
+ if profiles.get('items'):
+ return profiles.get('items')[0].get('id')
+
+ return None
+
--- /dev/null
+++ b/ckanext/ga_report/ga_model.py
@@ -1,1 +1,105 @@
+import re
+import uuid
+from sqlalchemy import Table, Column, MetaData
+from sqlalchemy import types
+from sqlalchemy.sql import select
+from sqlalchemy.orm import mapper
+from sqlalchemy import func
+
+import ckan.model as model
+from ckan.lib.base import *
+
+def make_uuid():
+ return unicode(uuid.uuid4())
+
+
+
+class GA_Url(object):
+
+ def __init__(self, **kwargs):
+ for k,v in kwargs.items():
+ setattr(self, k, v)
+
+
+metadata = MetaData()
+url_table = Table('ga_url', metadata,
+ Column('id', types.UnicodeText, primary_key=True,
+ default=make_uuid),
+ Column('period_name', types.UnicodeText),
+ Column('period_complete_day', types.Integer),
+ Column('metric', types.UnicodeText),
+ Column('value', types.UnicodeText),
+ Column('url', types.UnicodeText),
+ Column('department_id', types.UnicodeText),
+ )
+mapper(GA_Url, url_table)
+
+
+def init_tables():
+ metadata.create_all(model.meta.engine)
+
+
+cached_tables = {}
+
+
+def get_table(name):
+ if name not in cached_tables:
+ meta = MetaData()
+ meta.reflect(bind=model.meta.engine)
+ table = meta.tables[name]
+ cached_tables[name] = table
+ return cached_tables[name]
+
+
+def _normalize_url(url):
+ '''Strip off the hostname etc. Do this before storing it.
+
+ >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices')
+ '/dataset/weekly_fuel_prices'
+ '''
+ url = re.sub('https?://(www\.)?data.gov.uk', '', url)
+ return url
+
+
+def _get_department_id_of_url(url):
+ # e.g. /dataset/fuel_prices
+ # e.g. /dataset/fuel_prices/resource/e63380d4
+ dataset_match = re.match('/dataset/([^/]+)(/.*)?', url)
+ if dataset_match:
+ dataset_ref = dataset_match.groups()[0]
+ dataset = model.Package.get(dataset_ref)
+ if dataset:
+ publisher_groups = dataset.get_groups('publisher')
+ if publisher_groups:
+ return publisher_groups[0].name
+
+
+def update_url_stats(period_name, period_complete_day, url_data):
+ for url, views, next_page in url_data:
+ url = _normalize_url(url)
+ department_id = _get_department_id_of_url(url)
+
+ # see if the row for this url & month is in the table already
+ item = model.Session.query(GA_Url).\
+ filter(GA_Url.period_name==period_name).\
+ filter(GA_Url.url==url).\
+ filter(GA_Url.metric == 'Total views').first()
+ if item:
+ item.period_name = period_complete_day = period_complete_day
+ item.value = views
+ item.department_id = department_id
+ model.Session.add(item)
+ else:
+ # create the row
+ values = {'id': make_uuid(),
+ 'period_name': period_name,
+ 'period_complete_day': period_complete_day,
+ 'url': url,
+ 'value': views,
+ 'metric': 'Total views',
+ 'department_id': department_id
+ }
+ model.Session.add(GA_Url(**values))
+ model.Session.commit()
+
--- /dev/null
+++ b/ckanext/ga_report/plugin.py
@@ -1,1 +1,26 @@
+import logging
+import ckan.lib.helpers as h
+from ckan.plugins import implements, toolkit
+import gasnippet
+import commands
+import dbutil
+log = logging.getLogger('ckanext.ga-report')
+
+class GoogleAnalyticsPlugin(p.SingletonPlugin):
+ implements(p.IConfigurer, inherit=True)
+ implements(p.IRoutes, inherit=True)
+
+ def update_config(self, config):
+ toolkit.add_template_directory(config, 'templates')
+ toolkit.add_public_directory(config, 'public')
+
+ def after_map(self, map):
+ map.connect(
+ '/data/analytics/index',
+ controller='ckanext.ga-report.controller:GaReport',
+ action='index'
+ )
+ return map
+
+
--- /dev/null
+++ b/ckanext/ga_report/report_model.py
--- /dev/null
+++ b/ckanext/ga_report/tests/test_api.py
@@ -1,1 +1,51 @@
+import os
+import datetime
+from nose.tools import assert_equal
+from ckanext.ga_report.download_analytics import DownloadAnalytics
+from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
+from ckanext.ga_report.ga_model import init_tables
+class TestAPI:
+
+ @classmethod
+ def setup_class(cls):
+ if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
+ print '*' * 60
+ print "Tests may not run without first having run the auth process"
+ print '*' * 60
+ init_tables()
+
+ @classmethod
+ def teardown_class(cls):
+ pass
+
+ def test_latest(self):
+ svc = init_service("token.dat", "credentials.json")
+ try:
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+ downloader.latest()
+ except Exception as e:
+ assert False, e
+
+
+ def test_since(self):
+ svc = init_service("token.dat", "credentials.json")
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+ try:
+ downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30))
+ except Exception as e:
+ assert False, e
+
+"""
+ downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+
+ time_period = self.args[1] if self.args and len(self.args) > 1 \
+ else 'latest'
+ if time_period == 'all':
+ downloader.all_()
+ elif time_period == 'latest':
+ downloader.latest()
+ else:
+ since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
+ downloader.since_date(since_date)
+"""
--- /dev/null
+++ b/ckanext/ga_report/tests/test_auth.py
@@ -1,1 +1,41 @@
+import os
+from nose.tools import assert_equal
+from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
+class TestAuth:
+
+ @classmethod
+ def setup_class(cls):
+ if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
+ print '*' * 60
+ print "Tests may not run without first having run the auth process"
+ print '*' * 60
+
+ @classmethod
+ def teardown_class(cls):
+ pass
+
+ def test_init(self):
+ try:
+ res = init_service(None, None)
+ assert False, "Init service worked without credentials or tokens"
+ except TypeError:
+ pass
+
+ def test_init_with_token(self):
+ res = init_service("token.dat", None)
+ assert res is not None, "Init service worked without credentials"
+
+ def test_init_with_token_and_credentials(self):
+ res = init_service("token.dat", "credentials.json")
+ assert res is not None, "Unable to create service with valid details"
+
+ def test_init_with_redentials(self):
+ #res = init_service("", "credentials.json")
+ # Triggers the auth flow via the browser
+ pass
+
+ def test_get_profile(self):
+ svc = init_service("token.dat", "credentials.json")
+ profile = get_profile_id(svc)
+ assert profile is not None, "Unable to find a profile given configured UA id and user details"
--- /dev/null
+++ b/credentials.json.template
@@ -1,1 +1,11 @@
+{
+ "installed": {
+ "client_id": "",
+ "client_secret": "",
+ "redirect_uris": [""],
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://accounts.google.com/o/oauth2/token"
+ }
+}
+
--- /dev/null
+++ b/setup.py
@@ -1,1 +1,37 @@
+from setuptools import setup, find_packages
+import sys, os
+version = '0.1'
+
+setup(
+ name='ckanext-ga-report',
+ version=version,
+ description="GA reporting for CKAN",
+ long_description="""\
+ """,
+ classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+ keywords='',
+ author='David Read',
+ author_email='david.read@hackneyworkshop.com',
+ url='',
+ license='',
+ packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+ namespace_packages=['ckanext', 'ckanext.ga_report'],
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=[
+ 'gdata',
+ 'google-api-python-client'
+ ],
+ entry_points=\
+ """
+ [ckan.plugins]
+ # Add plugins here, eg
+ ga-report=ckanext.ga_report.plugin:GaReportPlugin
+
+ [paste.paster_command]
+ loadanalytics = ckanext.ga_report.command:LoadAnalytics
+ initdb = ckanext.ga_report.command:InitDB
+ """,
+)
+