From: Ross Jones <rossdjones@gmail.com>
Date: Wed, 10 Oct 2012 12:39:36 +0000
Subject: Remove prettyprinting unless DEBUG is set in env and add a new
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=da94cefe7c9c19db38d7c9f7e395d3c958b722fb
---
Remove prettyprinting unless DEBUG is set in env and add a new
test for actually making the calls.
---


--- a/README.rst
+++ b/README.rst
@@ -31,11 +31,10 @@
 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration::
 
       googleanalytics.id = UA-1010101-1
-      googleanalytics.username = googleaccount@gmail.com
-      googleanalytics.password = googlepassword
+      googleanalytics.account = Account name (i.e. data.gov.uk, see top level item at https://www.google.com/analytics)
       ga-report.period = monthly
 
-   Note that your password will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose.
+   Note that your credentials will be readable by system administrators on your server. Rather than use sensitive account details, it is suggested you give access to the GA account to a new Google account that you create just for this purpose.
 
 3. Set up this extension's database tables using a paster command. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file)::
 

--- a/ckanext/ga_report/command.py
+++ b/ckanext/ga_report/command.py
@@ -1,7 +1,10 @@
 import logging
+import datetime
 
 from ckan.lib.cli import CkanCommand
-# No other CKAN imports allowed until _load_config is run, or logging is disabled
+# No other CKAN imports allowed until _load_config is run,
+# or logging is disabled
+
 
 class InitDB(CkanCommand):
     """Initialise the extension's database tables
@@ -46,9 +49,10 @@
         assuming it is correct.
         """
         from ga_auth import init_service
-        initialize_service('token.dat',
-                           self.args[0] if self.args
-                                        else 'credentials.json')
+        init_service('token.dat',
+                      self.args[0] if self.args
+                                   else 'credentials.json')
+
 
 class LoadAnalytics(CkanCommand):
     """Get data from Google Analytics API and save it
@@ -73,18 +77,20 @@
     def command(self):
         self._load_config()
 
-        from ga_auth import init_service
+        from download_analytics import DownloadAnalytics
+        from ga_auth import (init_service, get_profile_id)
+
         try:
             svc = init_service(self.args[0], None)
         except TypeError:
-            print 'Have you correctly run the getauthtoken task and specified the correct file here'
+            print ('Have you correctly run the getauthtoken task and '
+                   'specified the correct file here')
             return
 
-        from download_analytics import DownloadAnalytics
-        from ga_auth import get_profile_id
         downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
 
-        time_period = self.args[1] if self.args and len(self.args) > 1 else 'latest'
+        time_period = self.args[1] if self.args and len(self.args) > 1 \
+            else 'latest'
         if time_period == 'all':
             downloader.all_()
         elif time_period == 'latest':
@@ -93,4 +99,3 @@
             since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
             downloader.since_date(since_date)
 
-

--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -1,3 +1,4 @@
+import os
 import logging
 import datetime
 
@@ -94,11 +95,7 @@
         '''Get data from GA for a given time period'''
         start_date = start_date.strftime('%Y-%m-%d')
         end_date = end_date.strftime('%Y-%m-%d')
-        # url
-        #query = 'ga:pagePath=~^%s,ga:pagePath=~^%s' % \
-        #        (PACKAGE_URL, self.resource_url_tag)
-        query = 'ga:pagePath=~^/dataset/'
-        #query = 'ga:pagePath=~^/User/'
+        query = 'ga:pagePath=~/dataset/[a-z0-9-]+$'
         metrics = 'ga:uniquePageviews'
         sort = '-ga:uniquePageviews'
 
@@ -110,33 +107,20 @@
                                  start_date=start_date,
                                  metrics=metrics,
                                  sort=sort,
+                                 dimensions="ga:pagePath",
+                                 max_results=10000,
                                  end_date=end_date).execute()
-        self.print_results(results)
 
-#        for entry in GA.ga_query(query_filter=query,
-#                                 from_date=start_date,
-#                                 metrics=metrics,
-#                                 sort=sort,
-#                                 to_date=end_date):
-#            print entry, type(entry)
-#            import pdb; pdb.set_trace()
-#            for dim in entry.dimension:
-#                if dim.name == "ga:pagePath":
-#                    package = dim.value
-#                    count = entry.get_metric(
-#                        'ga:uniquePageviews').value or 0
-#                    packages[package] = int(count)
-        return []
+        if os.getenv('DEBUG'):
+            import pprint
+            pprint.pprint(results)
+            print 'Total results: %s' % results.get('totalResults')
 
-    def print_results(self, results):
-        import pprint
-        pprint.pprint(results)
-        if results:
-            print 'Profile: %s' % results.get('profileInfo').get('profileName')
-            print 'Total results: %s' % results.get('totalResults')
-            print 'Total Visits: %s' % results.get('rows', [[-1]])[0][0]
-        else:
-            print 'No results found'
+        packages = []
+        for entry in results.get('rows'):
+            (loc,size,) = entry
+            packages.append( ('http:/' + loc,size, '',) ) # Temporary hack
+        return dict(url=packages)
 
     def store(self, period_name, period_complete_day, data):
         if 'url' in data:

--- a/ckanext/ga_report/ga_auth.py
+++ b/ckanext/ga_report/ga_auth.py
@@ -1,3 +1,4 @@
+import os
 import httplib2
 from apiclient.discovery import build
 from oauth2client.client import flow_from_clientsecrets
@@ -7,12 +8,12 @@
 from pylons import config
 
 
-def _prepare_credentials( token_filename, credentials_filename ):
+def _prepare_credentials(token_filename, credentials_filename):
     """
     Either returns the user's oauth credentials or uses the credentials
     file to generate a token (by forcing the user to login in the browser)
     """
-    storage = Storage( token_filename )
+    storage = Storage(token_filename)
     credentials = storage.get()
 
     if credentials is None or credentials.invalid:
@@ -23,7 +24,8 @@
 
     return credentials
 
-def init_service( token_file, credentials_file ):
+
+def init_service(token_file, credentials_file):
     """
     Given a file containing the user's oauth token (and another with
     credentials in case we need to generate the token) will return a
@@ -40,15 +42,24 @@
 def get_profile_id(service):
     """
     Get the profile ID for this user and the service specified by the
-    'googleanalytics.id' configuration option.
+    'googleanalytics.id' configuration option. This function iterates
+    over all of the accounts available to the user who invoked the
+    service to find one where the account name matches (in case the
+    user has several).
     """
     accounts = service.management().accounts().list().execute()
 
     if not accounts.get('items'):
         return None
 
-    accountId = accounts.get('items')[0].get('id')
+    accountName = config.get('googleanalytics.account')
     webPropertyId = config.get('googleanalytics.id')
+    for acc in accounts.get('items'):
+        if acc.get('name') == accountName:
+            accountId = acc.get('id')
+
+    webproperties = service.management().webproperties().list(accountId=accountId).execute()
+
     profiles = service.management().profiles().list(
         accountId=accountId, webPropertyId=webPropertyId).execute()
 
@@ -56,3 +67,4 @@
         return profiles.get('items')[0].get('id')
 
     return None
+

--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -3,30 +3,45 @@
 
 from sqlalchemy import Table, Column, MetaData
 from sqlalchemy import types
-from sqlalchemy.sql import select, text
+from sqlalchemy.sql import select
+from sqlalchemy.orm import mapper
 from sqlalchemy import func
 
 import ckan.model as model
-from ckan.model.types import JsonType
 from ckan.lib.base import *
 
 def make_uuid():
     return unicode(uuid.uuid4())
 
+
+
+class GA_Url(object):
+
+    def __init__(self, **kwargs):
+        for k,v in kwargs.items():
+            setattr(self, k, v)
+
+
+metadata = MetaData()
+url_table = Table('ga_url', metadata,
+                      Column('id', types.UnicodeText, primary_key=True,
+                             default=make_uuid),
+                      Column('period_name', types.UnicodeText),
+                      Column('period_complete_day', types.Integer),
+                      Column('metric', types.UnicodeText),
+                      Column('value', types.UnicodeText),
+                      Column('url', types.UnicodeText),
+                      Column('department_id', types.UnicodeText),
+                )
+mapper(GA_Url, url_table)
+
+
 def init_tables():
-    metadata = MetaData()
-    package_stats = Table('ga_url', metadata,
-                          Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
-                          Column('period_name', types.UnicodeText),
-                          Column('period_complete_day', types.Integer),
-                          Column('visits', types.Integer),
-                          Column('group_id', types.String(60)),
-                          Column('next_page', JsonType),
-                          )
     metadata.create_all(model.meta.engine)
 
 
 cached_tables = {}
+
 
 def get_table(name):
     if name not in cached_tables:
@@ -46,6 +61,7 @@
     url = re.sub('https?://(www\.)?data.gov.uk', '', url)
     return url
 
+
 def _get_department_id_of_url(url):
     # e.g. /dataset/fuel_prices
     # e.g. /dataset/fuel_prices/resource/e63380d4
@@ -56,36 +72,34 @@
         if dataset:
             publisher_groups = dataset.get_groups('publisher')
             if publisher_groups:
-                return publisher_groups[0].id
+                return publisher_groups[0].name
+
 
 def update_url_stats(period_name, period_complete_day, url_data):
-    table = get_table('ga_url')
-    connection = model.Session.connection()
     for url, views, next_page in url_data:
         url = _normalize_url(url)
         department_id = _get_department_id_of_url(url)
+
         # see if the row for this url & month is in the table already
-        s = select([func.count(id_col)],
-                   table.c.period_name == period_name,
-                   table.c.url == url)
-        count = connection.execute(s).fetchone()
-        if count and count[0]:
-            # update the row
-            connection.execute(table.update()\
-                .where(table.c.period_name == period_name,
-                       table.c.url == url)\
-                .values(period_complete_day=period_complete_day,
-                        views=views,
-                        department_id=department_id,
-                        next_page=next_page))
+        item = model.Session.query(GA_Url).\
+            filter(GA_Url.period_name==period_name).\
+            filter(GA_Url.url==url).\
+            filter(GA_Url.metric == 'Total views').first()
+        if item:
+            item.period_name = period_complete_day = period_complete_day
+            item.value = views
+            item.department_id = department_id
+            model.Session.add(item)
         else:
             # create the row
-            values = {'period_name': period_name,
+            values = {'id': make_uuid(),
+                      'period_name': period_name,
                       'period_complete_day': period_complete_day,
                       'url': url,
-                      'views': views,
-                      'department_id': department_id,
-                      'next_page': next_page}
-            connection.execute(stats.insert()\
-                               .values(**values))
+                      'value': views,
+                      'metric': 'Total views',
+                      'department_id': department_id
+                     }
+            model.Session.add(GA_Url(**values))
+        model.Session.commit()
 

--- /dev/null
+++ b/ckanext/ga_report/tests/test_api.py
@@ -1,1 +1,51 @@
+import os
+import datetime
+from nose.tools import assert_equal
+from ckanext.ga_report.download_analytics import DownloadAnalytics
+from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
+from ckanext.ga_report.ga_model import init_tables
 
+class TestAPI:
+
+    @classmethod
+    def setup_class(cls):
+        if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
+            print '*' * 60
+            print "Tests may not run without first having run the auth process"
+            print '*' * 60
+        init_tables()
+
+    @classmethod
+    def teardown_class(cls):
+        pass
+
+    def test_latest(self):
+        svc = init_service("token.dat", "credentials.json")
+        try:
+            downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+            downloader.latest()
+        except Exception as e:
+            assert False, e
+
+
+    def test_since(self):
+        svc = init_service("token.dat", "credentials.json")
+        downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+        try:
+            downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30))
+        except Exception as e:
+            assert False, e
+
+"""
+        downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
+
+        time_period = self.args[1] if self.args and len(self.args) > 1 \
+            else 'latest'
+        if time_period == 'all':
+            downloader.all_()
+        elif time_period == 'latest':
+            downloader.latest()
+        else:
+            since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
+            downloader.since_date(since_date)
+"""

--- /dev/null
+++ b/ckanext/ga_report/tests/test_auth.py
@@ -1,1 +1,41 @@
+import os
+from nose.tools import assert_equal
+from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
 
+class TestAuth:
+
+    @classmethod
+    def setup_class(cls):
+        if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
+            print '*' * 60
+            print "Tests may not run without first having run the auth process"
+            print '*' * 60
+
+    @classmethod
+    def teardown_class(cls):
+        pass
+
+    def test_init(self):
+        try:
+            res = init_service(None, None)
+            assert False, "Init service worked without credentials or tokens"
+        except TypeError:
+            pass
+
+    def test_init_with_token(self):
+        res = init_service("token.dat", None)
+        assert res is not None, "Init service worked without credentials"
+
+    def test_init_with_token_and_credentials(self):
+        res = init_service("token.dat", "credentials.json")
+        assert res is not None, "Unable to create service with valid details"
+
+    def test_init_with_redentials(self):
+        #res = init_service("", "credentials.json")
+        # Triggers the auth flow via the browser
+        pass
+
+    def test_get_profile(self):
+        svc = init_service("token.dat", "credentials.json")
+        profile = get_profile_id(svc)
+        assert profile is not None, "Unable to find a profile given configured UA id and user details"