From: Ross Jones Date: Wed, 07 Nov 2012 13:36:39 +0000 Subject: Changes to tidy up the handling of all months; X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=4efdb3cd382347567bfc3bb26346b00a6674df08 --- Changes to tidy up the handling of all months; --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -9,7 +9,7 @@ import sqlalchemy from sqlalchemy import func, cast, Integer import ckan.model as model -from ga_model import GA_Url, GA_Stat, GA_ReferralStat +from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher log = logging.getLogger('ckanext.ga-report') @@ -23,7 +23,7 @@ def _month_details(cls): months = [] - vals = model.Session.query(cls.period_name).distinct().all() + vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all() for m in vals: months.append( (m[0], _get_month_name(m[0]))) return sorted(months, key=operator.itemgetter(0), reverse=True) @@ -228,7 +228,6 @@ c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) c.top_publishers = _get_top_publishers() - return render('ga_report/publisher/index.html') def _get_packages(self, publisher=None, count=-1): @@ -236,43 +235,22 @@ if count == -1: count = sys.maxint - q = model.Session.query(GA_Url)\ + month = c.month or 'All' + + q = model.Session.query(GA_Url,model.Package)\ + .filter(model.Package.name==GA_Url.package_id)\ .filter(GA_Url.url.like('/dataset/%')) if publisher: q = q.filter(GA_Url.department_id==publisher.name) - if c.month: - q = q.filter(GA_Url.period_name==c.month) + q = q.filter(GA_Url.period_name==month) q = q.order_by('ga_url.visitors::int desc') - - if c.month: - top_packages = [] - for entry in q.limit(count): - package_name = entry.url[len('/dataset/'):] - p = model.Package.get(package_name) - if p: - top_packages.append((p, entry.pageviews, entry.visitors)) - else: - log.warning('Could not find package "%s"', package_name) - else: - ds = {} - for entry in q: - if len(ds) >= count: - break - package_name = entry.url[len('/dataset/'):] - p = model.Package.get(package_name) - if p: - if not p in ds: - ds[p] = {'views': 0, 'visits': 0} - ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) - ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) - else: - log.warning('Could not find package "%s"', package_name) - - results = [] - for k, v in ds.iteritems(): - results.append((k,v['views'],v['visits'])) - - top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + top_packages = [] + for entry,package in q.limit(count): + if package: + top_packages.append((package, entry.pageviews, entry.visitors)) + else: + log.warning('Could not find package associated package') + return top_packages def read(self): @@ -308,15 +286,12 @@ else: c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + month = c.month or 'All' c.publisher_page_views = 0 q = model.Session.query(GA_Url).\ filter(GA_Url.url=='/publisher/%s' % c.publisher_name) - if c.month: - entry = q.filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 - else: - for e in q.all(): - c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 c.top_packages = self._get_packages(c.publisher, 20) @@ -327,33 +302,26 @@ Returns a list of the top 20 publishers by dataset visits. (The number to show can be varied with 'limit') ''' + month = c.month or 'All' connection = model.Session.connection() q = """ select department_id, sum(pageviews::int) views, sum(visitors::int) visits from ga_url - where department_id <> ''""" - if c.month: - q = q + """ - and period_name=%s - """ - q = q + """ - group by department_id order by visits desc + where department_id <> '' + and period_name=%s + group by department_id order by visits desc """ if limit: q = q + " limit %s;" % (limit) - # Add this back (before and period_name =%s) if you want to ignore publisher - # homepage views - # and not url like '/publisher/%%' - top_publishers = [] - res = connection.execute(q, c.month) - + res = connection.execute(q, month) for row in res: g = model.Group.get(row[0]) if g: top_publishers.append((g, row[1], row[2])) return top_publishers + def _get_publishers(): ''' --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -3,7 +3,7 @@ import datetime import collections from pylons import config - +from ga_model import _normalize_url import ga_model #from ga_client import GA @@ -101,12 +101,17 @@ start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') + # Clean up the entries before we run this + ga_model.pre_update_url_stats(period_name) + + accountName = config.get('googleanalytics.account') + + data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName) log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) self.store(period_name, period_complete_day, data, ) - data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+') + data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName) log.info('Storing Publisher Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) self.store(period_name, period_complete_day, data,) @@ -115,6 +120,7 @@ self.sitewide_stats( period_name ) self.update_social_info(period_name, start_date, end_date) + def update_social_info(self, period_name, start_date, end_date): start_date = start_date.strftime('%Y-%m-%d') @@ -137,17 +143,16 @@ data = collections.defaultdict(list) rows = results.get('rows',[]) for row in rows: - from ga_model import _normalize_url data[_normalize_url(row[0])].append( (row[1], int(row[2]),) ) ga_model.update_social(period_name, data) - def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'): + def download(self, start_date, end_date, path=None): '''Get data from GA for a given time period''' start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') query = 'ga:pagePath=%s$' % path - metrics = 'ga:uniquePageviews, ga:visitors' + metrics = 'ga:uniquePageviews, ga:visits' sort = '-ga:uniquePageviews' # Supported query params at @@ -165,7 +170,10 @@ packages = [] for entry in results.get('rows'): (loc,pageviews,visits) = entry - packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack + url = _normalize_url('http:/' + loc) + if not url.startswith('/dataset/') and not url.startswith('/publisher/'): + continue + packages.append( (url, pageviews, visits,) ) # Temporary hack return dict(url=packages) def store(self, period_name, period_complete_day, data): @@ -209,7 +217,7 @@ results = self.service.data().ga().get( ids='ga:' + self.profile_id, start_date=start_date, - metrics='ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors', + metrics='ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visits', max_results=10000, end_date=end_date).execute() result_data = results.get('rows') --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -1,10 +1,10 @@ import re import uuid -from sqlalchemy import Table, Column, MetaData +from sqlalchemy import Table, Column, MetaData, ForeignKey from sqlalchemy import types from sqlalchemy.sql import select -from sqlalchemy.orm import mapper +from sqlalchemy.orm import mapper, relation from sqlalchemy import func import ckan.model as model @@ -14,8 +14,6 @@ return unicode(uuid.uuid4()) metadata = MetaData() - - class GA_Url(object): @@ -32,6 +30,7 @@ Column('visitors', types.UnicodeText), Column('url', types.UnicodeText), Column('department_id', types.UnicodeText), + Column('package_id', types.UnicodeText), ) mapper(GA_Url, url_table) @@ -157,34 +156,71 @@ model.Session.commit() +def update_url_stat_totals(period_name): + + """ + items = model.Session.query(GA_Url).\ + filter(GA_Url.period_name != "All").\ + filter(GA_Url.url==url).all() + values = {'id': make_uuid(), + 'period_name': "All", + 'period_complete_day': "0", + 'url': url, + 'pageviews': sum([int(x.pageviews) for x in items]), + 'visitors': sum([int(x.visitors) for x in items]), + 'department_id': department_id, + 'package_id': package + } + model.Session.add(GA_Url(**values)) + model.Session.commit() + """ + +def pre_update_url_stats(period_name): + model.Session.query(GA_Url).\ + filter(GA_Url.period_name==period_name).delete() + model.Session.query(GA_Url).\ + filter(GA_Url.period_name=='All').delete() + def update_url_stats(period_name, period_complete_day, url_data): + for url, views, visitors in url_data: - url = _normalize_url(url) department_id = _get_department_id_of_url(url) - # see if the row for this url & month is in the table already - item = model.Session.query(GA_Url).\ - filter(GA_Url.period_name==period_name).\ - filter(GA_Url.url==url).first() - if item: - item.period_name = period_name - item.pageviews = views - item.visitors = visitors - item.department_id = department_id - model.Session.add(item) - else: - # create the row + package = None + if url.startswith('/dataset/'): + package = url[len('/dataset/'):] + + values = {'id': make_uuid(), + 'period_name': period_name, + 'period_complete_day': period_complete_day, + 'url': url, + 'pageviews': views, + 'visitors': visitors, + 'department_id': department_id, + 'package_id': package + } + model.Session.add(GA_Url(**values)) + model.Session.commit() + + if package: + entries = model.Session.query(GA_Url).\ + filter(GA_Url.period_name!='All').\ + filter(GA_Url.url==url).all() values = {'id': make_uuid(), - 'period_name': period_name, - 'period_complete_day': period_complete_day, + 'period_name': 'All', + 'period_complete_day': 0, 'url': url, - 'pageviews': views, - 'visitors': visitors, - 'department_id': department_id + 'pageviews': sum([int(e.pageviews) for e in entries]), + 'visitors': sum([int(e.visitors) for e in entries]), + 'department_id': department_id, + 'package_id': package } model.Session.add(GA_Url(**values)) - model.Session.commit() + model.Session.commit() + + + def update_social(period_name, data): --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -41,15 +41,15 @@ + - + -
PublisherDataset Visits Dataset ViewsDataset Visits
${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name))} ${visits} ${views}${visits}
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html +++ b/ckanext/ga_report/templates/ga_report/publisher/read.html @@ -47,15 +47,15 @@ + - + -
DatasetVisits ViewsVisits
${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} ${visits} ${views}${visits}