From: Ross Jones Date: Wed, 07 Nov 2012 10:16:44 +0000 Subject: Fixes to index for publishers X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=248a1062f467939141440d4ee4c13a89f69f116a --- Fixes to index for publishers --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -23,7 +23,7 @@ def _month_details(cls): months = [] - vals = model.Session.query(cls.period_name).distinct().all() + vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all() for m in vals: months.append( (m[0], _get_month_name(m[0]))) return sorted(months, key=operator.itemgetter(0), reverse=True) @@ -236,43 +236,23 @@ if count == -1: count = sys.maxint - q = model.Session.query(GA_Url)\ + month = c.month or 'All' + + q = model.Session.query(GA_Url,model.Package)\ + .filter(model.Package.name==GA_Url.package_id)\ .filter(GA_Url.url.like('/dataset/%')) if publisher: q = q.filter(GA_Url.department_id==publisher.name) - if c.month: - q = q.filter(GA_Url.period_name==c.month) + q = q.filter(GA_Url.period_name==month) q = q.order_by('ga_url.visitors::int desc') - - if c.month: - top_packages = [] - for entry in q.limit(count): - package_name = entry.url[len('/dataset/'):] - p = model.Package.get(package_name) - if p: - top_packages.append((p, entry.pageviews, entry.visitors)) - else: - log.warning('Could not find package "%s"', package_name) - else: - ds = {} - for entry in q: - if len(ds) >= count: - break - package_name = entry.url[len('/dataset/'):] - p = model.Package.get(package_name) - if p: - if not p in ds: - ds[p] = {'views': 0, 'visits': 0} - ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) - ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) - else: - log.warning('Could not find package "%s"', package_name) - - results = [] - for k, v in ds.iteritems(): - results.append((k,v['views'],v['visits'])) - - top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) + top_packages = [] + + for entry,package in q.limit(count): + if package: + top_packages.append((package, entry.pageviews, entry.visitors)) + else: + log.warning('Could not find package associated package') + return top_packages def read(self): @@ -308,15 +288,12 @@ else: c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + month = c.mnth or 'All' c.publisher_page_views = 0 q = model.Session.query(GA_Url).\ filter(GA_Url.url=='/publisher/%s' % c.publisher_name) - if c.month: - entry = q.filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 - else: - for e in q.all(): - c.publisher_page_views = c.publisher_page_views + int(e.pageviews) + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 c.top_packages = self._get_packages(c.publisher, 20) --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -147,7 +147,7 @@ start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') query = 'ga:pagePath=%s$' % path - metrics = 'ga:uniquePageviews, ga:visitors' + metrics = 'ga:uniquePageviews, ga:visits' sort = '-ga:uniquePageviews' # Supported query params at @@ -209,7 +209,7 @@ results = self.service.data().ga().get( ids='ga:' + self.profile_id, start_date=start_date, - metrics='ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visitors', + metrics='ga:pageviewsPerVisit,ga:avgTimeOnSite,ga:percentNewVisits,ga:visits', max_results=10000, end_date=end_date).execute() result_data = results.get('rows') --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -1,10 +1,10 @@ import re import uuid -from sqlalchemy import Table, Column, MetaData +from sqlalchemy import Table, Column, MetaData, ForeignKey from sqlalchemy import types from sqlalchemy.sql import select -from sqlalchemy.orm import mapper +from sqlalchemy.orm import mapper, relation from sqlalchemy import func import ckan.model as model @@ -14,8 +14,6 @@ return unicode(uuid.uuid4()) metadata = MetaData() - - class GA_Url(object): @@ -32,6 +30,7 @@ Column('visitors', types.UnicodeText), Column('url', types.UnicodeText), Column('department_id', types.UnicodeText), + Column('package_id', types.UnicodeText), ) mapper(GA_Url, url_table) @@ -163,6 +162,10 @@ url = _normalize_url(url) department_id = _get_department_id_of_url(url) + package = None + if url.startswith('/dataset/'): + package = url[len('/dataset/'):] + # see if the row for this url & month is in the table already item = model.Session.query(GA_Url).\ filter(GA_Url.period_name==period_name).\ @@ -172,6 +175,7 @@ item.pageviews = views item.visitors = visitors item.department_id = department_id + item.package_id = package model.Session.add(item) else: # create the row @@ -181,9 +185,31 @@ 'url': url, 'pageviews': views, 'visitors': visitors, - 'department_id': department_id + 'department_id': department_id, + 'package_id': package } model.Session.add(GA_Url(**values)) + + # We now need to recaculate the ALL time_period from the data we have + # Delete the old 'All' + old = model.Session.query(GA_Url).\ + filter(GA_Url.period_name == "All").\ + filter(GA_Url.url==url).delete() + + items = model.Session.query(GA_Url).\ + filter(GA_Url.period_name != "All").\ + filter(GA_Url.url==url).all() + values = {'id': make_uuid(), + 'period_name': "All", + 'period_complete_day': "0", + 'url': url, + 'pageviews': sum([int(x.pageviews) for x in items]), + 'visitors': sum([int(x.visitors) for x in items]), + 'department_id': department_id, + 'package_id': package + } + model.Session.add(GA_Url(**values)) + model.Session.commit()