projects / ckanext-ga-report.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Remove prettyprinting unless DEBUG is set in env and add a new
Remove prettyprinting unless DEBUG is set in env and add a new
test for actually making the calls.

file:a/ckanext/ga_report/download_analytics.py -> file:b/ckanext/ga_report/download_analytics.py
  import os
import logging import logging
import datetime import datetime
   
from pylons import config from pylons import config
   
import ga_model import ga_model
   
#from ga_client import GA #from ga_client import GA
   
log = logging.getLogger('ckanext.ga-report') log = logging.getLogger('ckanext.ga-report')
   
FORMAT_MONTH = '%Y-%m' FORMAT_MONTH = '%Y-%m'
   
class DownloadAnalytics(object): class DownloadAnalytics(object):
'''Downloads and stores analytics info''' '''Downloads and stores analytics info'''
   
def __init__(self, service=None, profile_id=None): def __init__(self, service=None, profile_id=None):
self.period = config['ga-report.period'] self.period = config['ga-report.period']
self.service = service self.service = service
self.profile_id = profile_id self.profile_id = profile_id
   
   
def all_(self): def all_(self):
self.since_date(datetime.datetime(2010, 1, 1)) self.since_date(datetime.datetime(2010, 1, 1))
   
def latest(self): def latest(self):
if self.period == 'monthly': if self.period == 'monthly':
# from first of this month to today # from first of this month to today
now = datetime.datetime.now() now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1) first_of_this_month = datetime.datetime(now.year, now.month, 1)
periods = ((now.strftime(FORMAT_MONTH), periods = ((now.strftime(FORMAT_MONTH),
now.day, now.day,
first_of_this_month, now),) first_of_this_month, now),)
else: else:
raise NotImplementedError raise NotImplementedError
self.download_and_store(periods) self.download_and_store(periods)
   
   
def since_date(self, since_date): def since_date(self, since_date):
assert isinstance(since_date, datetime.datetime) assert isinstance(since_date, datetime.datetime)
periods = [] # (period_name, period_complete_day, start_date, end_date) periods = [] # (period_name, period_complete_day, start_date, end_date)
if self.period == 'monthly': if self.period == 'monthly':
first_of_the_months_until_now = [] first_of_the_months_until_now = []
year = since_date.year year = since_date.year
month = since_date.month month = since_date.month
now = datetime.datetime.now() now = datetime.datetime.now()
first_of_this_month = datetime.datetime(now.year, now.month, 1) first_of_this_month = datetime.datetime(now.year, now.month, 1)
while True: while True:
first_of_the_month = datetime.datetime(year, month, 1) first_of_the_month = datetime.datetime(year, month, 1)
if first_of_the_month == first_of_this_month: if first_of_the_month == first_of_this_month:
periods.append((now.strftime(FORMAT_MONTH), periods.append((now.strftime(FORMAT_MONTH),
now.day, now.day,
first_of_this_month, now)) first_of_this_month, now))
break break
elif first_of_the_month < first_of_this_month: elif first_of_the_month < first_of_this_month:
in_the_next_month = first_of_the_month + datetime.timedelta(40) in_the_next_month = first_of_the_month + datetime.timedelta(40)
last_of_the_month = datetime.datetime(in_the_next_month.year, last_of_the_month = datetime.datetime(in_the_next_month.year,
in_the_next_month.month, 1)\ in_the_next_month.month, 1)\
- datetime.timedelta(1) - datetime.timedelta(1)
periods.append((now.strftime(FORMAT_MONTH), 0, periods.append((now.strftime(FORMAT_MONTH), 0,
first_of_the_month, last_of_the_month)) first_of_the_month, last_of_the_month))
else: else:
# first_of_the_month has got to the future somehow # first_of_the_month has got to the future somehow
break break
month += 1 month += 1
if month > 12: if month > 12:
year += 1 year += 1
month = 1 month = 1
else: else:
raise NotImplementedError raise NotImplementedError
self.download_and_store(periods) self.download_and_store(periods)
   
@staticmethod @staticmethod
def get_full_period_name(period_name, period_complete_day): def get_full_period_name(period_name, period_complete_day):
if period_complete_day: if period_complete_day:
return period_name + ' (up to %ith)' % period_complete_day return period_name + ' (up to %ith)' % period_complete_day
else: else:
return period_name return period_name
   
   
def download_and_store(self, periods): def download_and_store(self, periods):
for period_name, period_complete_day, start_date, end_date in periods: for period_name, period_complete_day, start_date, end_date in periods:
log.info('Downloading Analytics for period "%s" (%s - %s)', log.info('Downloading Analytics for period "%s" (%s - %s)',
self.get_full_period_name(period_name, period_complete_day), self.get_full_period_name(period_name, period_complete_day),
start_date.strftime('%Y %m %d'), start_date.strftime('%Y %m %d'),
end_date.strftime('%Y %m %d')) end_date.strftime('%Y %m %d'))
data = self.download(start_date, end_date) data = self.download(start_date, end_date)
log.info('Storing Analytics for period "%s"', log.info('Storing Analytics for period "%s"',
self.get_full_period_name(period_name, period_complete_day)) self.get_full_period_name(period_name, period_complete_day))
self.store(period_name, period_complete_day, data) self.store(period_name, period_complete_day, data)
   
   
def download(self, start_date, end_date): def download(self, start_date, end_date):
'''Get data from GA for a given time period''' '''Get data from GA for a given time period'''
start_date = start_date.strftime('%Y-%m-%d') start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d')
query = 'ga:pagePath=~/dataset/[a-z0-9-]+$' query = 'ga:pagePath=~/dataset/[a-z0-9-]+$'
metrics = 'ga:uniquePageviews' metrics = 'ga:uniquePageviews'
sort = '-ga:uniquePageviews' sort = '-ga:uniquePageviews'
   
# Supported query params at # Supported query params at
# https://developers.google.com/analytics/devguides/reporting/core/v3/reference # https://developers.google.com/analytics/devguides/reporting/core/v3/reference
results = self.service.data().ga().get( results = self.service.data().ga().get(
ids='ga:' + self.profile_id, ids='ga:' + self.profile_id,
filters=query, filters=query,
start_date=start_date, start_date=start_date,
metrics=metrics, metrics=metrics,
sort=sort, sort=sort,
dimensions="ga:pagePath", dimensions="ga:pagePath",
max_results=10000, max_results=10000,
end_date=end_date).execute() end_date=end_date).execute()
   
  if os.getenv('DEBUG'):
import pprint import pprint
pprint.pprint(results) pprint.pprint(results)
print 'Total results: %s' % results.get('totalResults') print 'Total results: %s' % results.get('totalResults')
   
packages = [] packages = []
for entry in results.get('rows'): for entry in results.get('rows'):
(loc,size,) = entry (loc,size,) = entry
packages.append( ('http:/' + loc,size, '',) ) # Temporary hack packages.append( ('http:/' + loc,size, '',) ) # Temporary hack
return dict(url=packages) return dict(url=packages)
   
def store(self, period_name, period_complete_day, data): def store(self, period_name, period_complete_day, data):
if 'url' in data: if 'url' in data:
ga_model.update_url_stats(period_name, period_complete_day, data['url']) ga_model.update_url_stats(period_name, period_complete_day, data['url'])
   
file:a/ckanext/ga_report/ga_model.py -> file:b/ckanext/ga_report/ga_model.py
import re import re
import uuid import uuid
   
from sqlalchemy import Table, Column, MetaData from sqlalchemy import Table, Column, MetaData
from sqlalchemy import types from sqlalchemy import types
from sqlalchemy.sql import select from sqlalchemy.sql import select
from sqlalchemy.orm import mapper from sqlalchemy.orm import mapper
from sqlalchemy import func from sqlalchemy import func
   
import ckan.model as model import ckan.model as model
from ckan.lib.base import * from ckan.lib.base import *
   
def make_uuid(): def make_uuid():
return unicode(uuid.uuid4()) return unicode(uuid.uuid4())
   
   
   
class GA_Url(object): class GA_Url(object):
   
def __init__(self, **kwargs): def __init__(self, **kwargs):
for k,v in kwargs.items(): for k,v in kwargs.items():
setattr(self, k, v) setattr(self, k, v)
   
   
metadata = MetaData() metadata = MetaData()
url_table = Table('ga_url', metadata, url_table = Table('ga_url', metadata,
Column('id', types.UnicodeText, primary_key=True, Column('id', types.UnicodeText, primary_key=True,
default=make_uuid), default=make_uuid),
Column('period_name', types.UnicodeText), Column('period_name', types.UnicodeText),
Column('period_complete_day', types.Integer), Column('period_complete_day', types.Integer),
Column('metric', types.UnicodeText), Column('metric', types.UnicodeText),
Column('value', types.UnicodeText), Column('value', types.UnicodeText),
Column('url', types.UnicodeText), Column('url', types.UnicodeText),
Column('department_id', types.UnicodeText), Column('department_id', types.UnicodeText),
) )
mapper(GA_Url, url_table) mapper(GA_Url, url_table)
   
   
def init_tables(): def init_tables():
metadata.create_all(model.meta.engine) metadata.create_all(model.meta.engine)
   
   
cached_tables = {} cached_tables = {}
   
   
def get_table(name): def get_table(name):
if name not in cached_tables: if name not in cached_tables:
meta = MetaData() meta = MetaData()
meta.reflect(bind=model.meta.engine) meta.reflect(bind=model.meta.engine)
table = meta.tables[name] table = meta.tables[name]
cached_tables[name] = table cached_tables[name] = table
return cached_tables[name] return cached_tables[name]
   
   
def _normalize_url(url): def _normalize_url(url):
'''Strip off the hostname etc. Do this before storing it. '''Strip off the hostname etc. Do this before storing it.
   
>>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices') >>> normalize_url('http://data.gov.uk/dataset/weekly_fuel_prices')
'/dataset/weekly_fuel_prices' '/dataset/weekly_fuel_prices'
''' '''
url = re.sub('https?://(www\.)?data.gov.uk', '', url) url = re.sub('https?://(www\.)?data.gov.uk', '', url)
return url return url
   
   
def _get_department_id_of_url(url): def _get_department_id_of_url(url):
# e.g. /dataset/fuel_prices # e.g. /dataset/fuel_prices
# e.g. /dataset/fuel_prices/resource/e63380d4 # e.g. /dataset/fuel_prices/resource/e63380d4
dataset_match = re.match('/dataset/([^/]+)(/.*)?', url) dataset_match = re.match('/dataset/([^/]+)(/.*)?', url)
if dataset_match: if dataset_match:
dataset_ref = dataset_match.groups()[0] dataset_ref = dataset_match.groups()[0]
dataset = model.Package.get(dataset_ref) dataset = model.Package.get(dataset_ref)
if dataset: if dataset:
publisher_groups = dataset.get_groups('publisher') publisher_groups = dataset.get_groups('publisher')
if publisher_groups: if publisher_groups:
return publisher_groups[0].name return publisher_groups[0].name
   
   
def update_url_stats(period_name, period_complete_day, url_data): def update_url_stats(period_name, period_complete_day, url_data):
table = get_table('ga_url')  
for url, views, next_page in url_data: for url, views, next_page in url_data:
url = _normalize_url(url) url = _normalize_url(url)
department_id = _get_department_id_of_url(url) department_id = _get_department_id_of_url(url)
   
# see if the row for this url & month is in the table already # see if the row for this url & month is in the table already
item = model.Session.query(GA_Url).\ item = model.Session.query(GA_Url).\
filter(GA_Url.period_name==period_name).\ filter(GA_Url.period_name==period_name).\
filter(GA_Url.url==url).\ filter(GA_Url.url==url).\
filter(GA_Url.metric == 'Total views').first() filter(GA_Url.metric == 'Total views').first()
if item: if item:
item.period_name = period_complete_day = period_complete_day item.period_name = period_complete_day = period_complete_day
item.value = views item.value = views
item.department_id = department_id item.department_id = department_id
model.Session.add(item) model.Session.add(item)
else: else:
# create the row # create the row
values = {'id': make_uuid(), values = {'id': make_uuid(),
'period_name': period_name, 'period_name': period_name,
'period_complete_day': period_complete_day, 'period_complete_day': period_complete_day,
'url': url, 'url': url,
'value': views, 'value': views,
'metric': 'Total views', 'metric': 'Total views',
'department_id': department_id 'department_id': department_id
} }
model.Session.add(GA_Url(**values)) model.Session.add(GA_Url(**values))
model.Session.commit() model.Session.commit()
   
file:b/ckanext/ga_report/tests/test_api.py (new)
  import os
  import datetime
  from nose.tools import assert_equal
  from ckanext.ga_report.download_analytics import DownloadAnalytics
  from ckanext.ga_report.ga_auth import (init_service, get_profile_id)
  from ckanext.ga_report.ga_model import init_tables
 
  class TestAPI:
 
  @classmethod
  def setup_class(cls):
  if not os.path.exists("token.dat") or not os.path.exists("credentials.json"):
  print '*' * 60
  print "Tests may not run without first having run the auth process"
  print '*' * 60
  init_tables()
 
  @classmethod
  def teardown_class(cls):
  pass
 
  def test_latest(self):
  svc = init_service("token.dat", "credentials.json")
  try:
  downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
  downloader.latest()
  except Exception as e:
  assert False, e
 
 
  def test_since(self):
  svc = init_service("token.dat", "credentials.json")
  downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
  try:
  downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30))
  except Exception as e:
  assert False, e
 
  """
  downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
 
  time_period = self.args[1] if self.args and len(self.args) > 1 \
  else 'latest'
  if time_period == 'all':
  downloader.all_()
  elif time_period == 'latest':
  downloader.latest()
  else:
  since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d')
  downloader.since_date(since_date)
  """
Unnamed repository; edit this file 'description' to name the repository.
GitPHP by Chris Han