From: Ross Jones Date: Tue, 09 Oct 2012 12:37:05 +0000 Subject: flake8 X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=8f3c8b58e7fd5e936d3e525ca0d6b52919d15635 --- flake8 --- --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ develop-eggs .installed.cfg +# Private info +credentials.json +token.dat + # Installer logs pip-log.txt --- a/README.rst +++ b/README.rst @@ -46,12 +46,52 @@ ckan.plugins = ga-report +Authorization +-------------- + +Before you can access the data, you need to set up the OAUTH details which you can do by following the `instructions `_ the outcome of which will be a file called credentials.json which should look like credentials.json.template with the relevant fields completed. These steps are below for convenience: + +1. Visit the `Google APIs Console `_ + +2. Sign-in and create a project or use an existing project. + +3. In the `Services pane `_ , activate Analytics API for your project. If prompted, read and accept the terms of service. + +4. Go to the `API Access pane `_ + +5. Click Create an OAuth 2.0 client ID.... + +6. Fill out the Branding Information fields and click Next. + +7. In Client ID Settings, set Application type to Installed application. + +8. Click Create client ID + +9. The details you need below are Client ID, Client secret, and Redirect URIs + + +Once you have set up your credentials.json file you can generate an oauth token file by using the +following command, which will store your oauth token in a file called token.dat once you have finished +giving permission in the browser:: + + $ paster getauthtoken --config=../ckan/development.ini + + Tutorial -------- -Download some GA data and store it in CKAN's db. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file):: +Download some GA data and store it in CKAN's db. (Ensure your CKAN pyenv is still activated, run the command from ``src/ckanext-ga-report``, alter the ``--config`` option to point to your site config file) and specifying the name of your auth file (token.dat by default) from the previous step:: - $ paster loadanalytics latest --config=../ckan/development.ini + $ paster loadanalytics token.dat latest --config=../ckan/development.ini + +The value after the token file is how much data you want to retrieve, this can be + +* **all** - data for all time (since 2010) + +* **latest** - (default) just the 'latest' data + +* **YYYY-MM-DD** - just data for all time periods going back to (and including) this date + Software Licence --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -1,7 +1,10 @@ import logging +import datetime from ckan.lib.cli import CkanCommand -# No other CKAN imports allowed until _load_config is run, or logging is disabled +# No other CKAN imports allowed until _load_config is run, +# or logging is disabled + class InitDB(CkanCommand): """Initialise the extension's database tables @@ -23,13 +26,44 @@ ga_model.init_tables() log.info("DB tables are setup") + +class GetAuthToken(CkanCommand): + """ Get's the Google auth token + + Usage: paster getauthtoken + + Where is the file name containing the details + for the service (obtained from https://code.google.com/apis/console). + By default this is set to credentials.json + """ + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = 0 + min_args = 0 + + def command(self): + """ + In this case we don't want a valid service, but rather just to + force the user through the auth flow. We allow this to complete to + act as a form of verification instead of just getting the token and + assuming it is correct. + """ + from ga_auth import init_service + init_service('token.dat', + self.args[0] if self.args + else 'credentials.json') + + class LoadAnalytics(CkanCommand): """Get data from Google Analytics API and save it in the ga_model - Usage: paster loadanalytics + Usage: paster loadanalytics - Where is: + Where is the name of the auth token file from + the getauthtoken step. + + And where is: all - data for all time latest - (default) just the 'latest' data YYYY-MM-DD - just data for all time periods going @@ -37,16 +71,26 @@ """ summary = __doc__.split('\n')[0] usage = __doc__ - max_args = 1 - min_args = 0 + max_args = 2 + min_args = 1 def command(self): self._load_config() from download_analytics import DownloadAnalytics - downloader = DownloadAnalytics() - - time_period = self.args[0] if self.args else 'latest' + from ga_auth import (init_service, get_profile_id) + + try: + svc = init_service(self.args[0], None) + except TypeError: + print ('Have you correctly run the getauthtoken task and ' + 'specified the correct file here') + return + + downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) + + time_period = self.args[1] if self.args and len(self.args) > 1 \ + else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': @@ -55,4 +99,3 @@ since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d') downloader.since_date(since_date) - --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -4,7 +4,8 @@ from pylons import config import ga_model -from ga_client import GA + +#from ga_client import GA log = logging.getLogger('ckanext.ga-report') @@ -12,12 +13,16 @@ class DownloadAnalytics(object): '''Downloads and stores analytics info''' - def __init__(self): + + def __init__(self, service=None, profile_id=None): self.period = config['ga-report.period'] - + self.service = service + self.profile_id = profile_id + + def all_(self): - pass - + self.since_date(datetime.datetime(2010, 1, 1)) + def latest(self): if self.period == 'monthly': # from first of this month to today @@ -49,8 +54,8 @@ break elif first_of_the_month < first_of_this_month: in_the_next_month = first_of_the_month + datetime.timedelta(40) - last_of_the_month == datetime.datetime(in_the_next_month.year, - in_the_next_month.month, a)\ + last_of_the_month = datetime.datetime(in_the_next_month.year, + in_the_next_month.month, 1)\ - datetime.timedelta(1) periods.append((now.strftime(FORMAT_MONTH), 0, first_of_the_month, last_of_the_month)) @@ -71,7 +76,7 @@ return period_name + ' (up to %ith)' % period_complete_day else: return period_name - + def download_and_store(self, periods): for period_name, period_complete_day, start_date, end_date in periods: @@ -84,8 +89,8 @@ self.get_full_period_name(period_name, period_complete_day)) self.store(period_name, period_complete_day, data) - @classmethod - def download(cls, start_date, end_date): + + def download(self, start_date, end_date): '''Get data from GA for a given time period''' start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') @@ -93,22 +98,45 @@ #query = 'ga:pagePath=~^%s,ga:pagePath=~^%s' % \ # (PACKAGE_URL, self.resource_url_tag) query = 'ga:pagePath=~^/dataset/' + #query = 'ga:pagePath=~^/User/' metrics = 'ga:uniquePageviews' sort = '-ga:uniquePageviews' - for entry in GA.ga_query(query_filter=query, - from_date=start_date, + + # Supported query params at + # https://developers.google.com/analytics/devguides/reporting/core/v3/reference + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + filters=query, + start_date=start_date, metrics=metrics, sort=sort, - to_date=end_date): - print entry - import pdb; pdb.set_trace() - for dim in entry.dimension: - if dim.name == "ga:pagePath": - package = dim.value - count = entry.get_metric( - 'ga:uniquePageviews').value or 0 - packages[package] = int(count) - return packages + end_date=end_date).execute() + self.print_results(results) + +# for entry in GA.ga_query(query_filter=query, +# from_date=start_date, +# metrics=metrics, +# sort=sort, +# to_date=end_date): +# print entry, type(entry) +# import pdb; pdb.set_trace() +# for dim in entry.dimension: +# if dim.name == "ga:pagePath": +# package = dim.value +# count = entry.get_metric( +# 'ga:uniquePageviews').value or 0 +# packages[package] = int(count) + return [] + + def print_results(self, results): + import pprint + pprint.pprint(results) + if results: + print 'Profile: %s' % results.get('profileInfo').get('profileName') + print 'Total results: %s' % results.get('totalResults') + print 'Total Visits: %s' % results.get('rows', [[-1]])[0][0] + else: + print 'No results found' def store(self, period_name, period_complete_day, data): if 'url' in data: --- /dev/null +++ b/ckanext/ga_report/ga_auth.py @@ -1,1 +1,60 @@ +import httplib2 +from apiclient.discovery import build +from oauth2client.client import flow_from_clientsecrets +from oauth2client.file import Storage +from oauth2client.tools import run +from pylons import config + + +def _prepare_credentials(token_filename, credentials_filename): + """ + Either returns the user's oauth credentials or uses the credentials + file to generate a token (by forcing the user to login in the browser) + """ + storage = Storage(token_filename) + credentials = storage.get() + + if credentials is None or credentials.invalid: + flow = flow_from_clientsecrets(credentials_filename, + scope='https://www.googleapis.com/auth/analytics.readonly', + message="Can't find the credentials file") + credentials = run(flow, storage) + + return credentials + + +def init_service(token_file, credentials_file): + """ + Given a file containing the user's oauth token (and another with + credentials in case we need to generate the token) will return a + service object representing the analytics API. + """ + http = httplib2.Http() + + credentials = _prepare_credentials(token_file, credentials_file) + http = credentials.authorize(http) # authorize the http object + + return build('analytics', 'v3', http=http) + + +def get_profile_id(service): + """ + Get the profile ID for this user and the service specified by the + 'googleanalytics.id' configuration option. + """ + accounts = service.management().accounts().list().execute() + + if not accounts.get('items'): + return None + + accountId = accounts.get('items')[0].get('id') + webPropertyId = config.get('googleanalytics.id') + profiles = service.management().profiles().list( + accountId=accountId, webPropertyId=webPropertyId).execute() + + if profiles.get('items'): + return profiles.get('items')[0].get('id') + + return None + --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -3,20 +3,23 @@ from sqlalchemy import Table, Column, MetaData from sqlalchemy import types -from sqlalchemy.sql import select, text +from sqlalchemy.sql import select from sqlalchemy import func import ckan.model as model from ckan.model.types import JsonType from ckan.lib.base import * + def make_uuid(): return unicode(uuid.uuid4()) + def init_tables(): metadata = MetaData() package_stats = Table('ga_url', metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), + Column('id', types.UnicodeText, primary_key=True, + default=make_uuid), Column('period_name', types.UnicodeText), Column('period_complete_day', types.Integer), Column('visits', types.Integer), @@ -27,6 +30,7 @@ cached_tables = {} + def get_table(name): if name not in cached_tables: @@ -46,6 +50,7 @@ url = re.sub('https?://(www\.)?data.gov.uk', '', url) return url + def _get_department_id_of_url(url): # e.g. /dataset/fuel_prices # e.g. /dataset/fuel_prices/resource/e63380d4 @@ -57,6 +62,7 @@ publisher_groups = dataset.get_groups('publisher') if publisher_groups: return publisher_groups[0].id + def update_url_stats(period_name, period_complete_day, url_data): table = get_table('ga_url') @@ -71,9 +77,9 @@ count = connection.execute(s).fetchone() if count and count[0]: # update the row - connection.execute(table.update()\ + connection.execute(table.update() .where(table.c.period_name == period_name, - table.c.url == url)\ + table.c.url == url) .values(period_complete_day=period_complete_day, views=views, department_id=department_id, @@ -86,6 +92,6 @@ 'views': views, 'department_id': department_id, 'next_page': next_page} - connection.execute(stats.insert()\ - .values(**values)) + connection.execute(stats.insert(). + values(**values)) --- /dev/null +++ b/credentials.json.template @@ -1,1 +1,11 @@ +{ + "installed": { + "client_id": "", + "client_secret": "", + "redirect_uris": [""], + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token" + } +} + --- a/setup.py +++ b/setup.py @@ -20,7 +20,8 @@ include_package_data=True, zip_safe=False, install_requires=[ - 'gdata' + 'gdata', + 'google-api-python-client' ], entry_points=\ """