Merge branch 'master' of github.com:datagovuk/ckanext-ga-report
Merge branch 'master' of github.com:datagovuk/ckanext-ga-report

--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -9,7 +9,7 @@
 import sqlalchemy
 from sqlalchemy import func, cast, Integer
 import ckan.model as model
-from ga_model import GA_Url, GA_Stat, GA_ReferralStat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher
 
 log = logging.getLogger('ckanext.ga-report')
 
@@ -22,8 +22,9 @@
 
 
 def _month_details(cls):
+    '''Returns a list of all the month names'''
     months = []
-    vals = model.Session.query(cls.period_name).distinct().all()
+    vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all()
     for m in vals:
         months.append( (m[0], _get_month_name(m[0])))
     return sorted(months, key=operator.itemgetter(0), reverse=True)
@@ -228,7 +229,6 @@
             c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
 
         c.top_publishers = _get_top_publishers()
-
         return render('ga_report/publisher/index.html')
 
     def _get_packages(self, publisher=None, count=-1):
@@ -236,41 +236,22 @@
         if count == -1:
             count = sys.maxint
 
-        q = model.Session.query(GA_Url)\
+        month = c.month or 'All'
+
+        q = model.Session.query(GA_Url,model.Package)\
+            .filter(model.Package.name==GA_Url.package_id)\
             .filter(GA_Url.url.like('/dataset/%'))
         if publisher:
             q = q.filter(GA_Url.department_id==publisher.name)
-        if c.month:
-            q = q.filter(GA_Url.period_name==c.month)
+        q = q.filter(GA_Url.period_name==month)
         q = q.order_by('ga_url.visitors::int desc')
-
-        if c.month:
-            top_packages = []
-            for entry in q.limit(count):
-                package_name = entry.url[len('/dataset/'):]
-                p = model.Package.get(package_name)
-                if p:
-                    top_packages.append((p, entry.pageviews, entry.visitors))
-                else:
-                    log.warning('Could not find package "%s"', package_name)
-        else:
-            ds = {}
-            for entry in q.limit(count):
-                package_name = entry.url[len('/dataset/'):]
-                p = model.Package.get(package_name)
-                if p:
-                    if not p in ds:
-                        ds[p] = {'views': 0, 'visits': 0}
-                    ds[p]['views'] = ds[p]['views'] + int(entry.pageviews)
-                    ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors)
-                else:
-                    log.warning('Could not find package "%s"', package_name)
-
-            results = []
-            for k, v in ds.iteritems():
-                results.append((k,v['views'],v['visits']))
-
-            top_packages = sorted(results, key=operator.itemgetter(1), reverse=True)
+        top_packages = []
+        for entry,package in q.limit(count):
+            if package:
+                top_packages.append((package, entry.pageviews, entry.visitors))
+            else:
+                log.warning('Could not find package associated package')
+
         return top_packages
 
     def read(self):
@@ -306,15 +287,12 @@
         else:
             c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
 
+        month = c.month or 'All'
         c.publisher_page_views = 0
         q = model.Session.query(GA_Url).\
             filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
-        if c.month:
-            entry = q.filter(GA_Url.period_name==c.month).first()
-            c.publisher_page_views = entry.pageviews if entry else 0
-        else:
-            for e in q.all():
-                c.publisher_page_views = c.publisher_page_views  + int(e.pageviews)
+        entry = q.filter(GA_Url.period_name==c.month).first()
+        c.publisher_page_views = entry.pageviews if entry else 0
 
         c.top_packages = self._get_packages(c.publisher, 20)
 
@@ -325,33 +303,26 @@
     Returns a list of the top 20 publishers by dataset visits.
     (The number to show can be varied with 'limit')
     '''
+    month = c.month or 'All'
     connection = model.Session.connection()
     q = """
         select department_id, sum(pageviews::int) views, sum(visitors::int) visits
         from ga_url
-        where department_id <> ''"""
-    if c.month:
-        q = q + """
-                and period_name=%s
-        """
-    q = q + """
-            group by department_id order by visits desc
+        where department_id <> ''
+          and period_name=%s
+        group by department_id order by visits desc
         """
     if limit:
         q = q + " limit %s;" % (limit)
 
-    # Add this back (before and period_name =%s) if you want to ignore publisher
-    # homepage views
-    # and not url like '/publisher/%%'
-
     top_publishers = []
-    res = connection.execute(q, c.month)
-
+    res = connection.execute(q, month)
     for row in res:
         g = model.Group.get(row[0])
         if g:
             top_publishers.append((g, row[1], row[2]))
     return top_publishers
+
 
 def _get_publishers():
     '''

--- a/ckanext/ga_report/download_analytics.py
+++ b/ckanext/ga_report/download_analytics.py
@@ -3,7 +3,7 @@
 import datetime
 import collections
 from pylons import config
-
+from ga_model import _normalize_url
 import ga_model
 
 #from ga_client import GA
@@ -101,12 +101,17 @@
                      start_date.strftime('%Y %m %d'),
                      end_date.strftime('%Y %m %d'))
 
-            data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+')
+            # Clean up the entries before we run this
+            ga_model.pre_update_url_stats(period_name)
+
+            accountName = config.get('googleanalytics.account')
+
+            data = self.download(start_date, end_date, '~/%s/dataset/[a-z0-9-_]+' % accountName)
             log.info('Storing Dataset Analytics for period "%s"',
                      self.get_full_period_name(period_name, period_complete_day))
             self.store(period_name, period_complete_day, data, )
 
-            data = self.download(start_date, end_date, '~/publisher/[a-z0-9-_]+')
+            data = self.download(start_date, end_date, '~/%s/publisher/[a-z0-9-_]+' % accountName)
             log.info('Storing Publisher Analytics for period "%s"',
                      self.get_full_period_name(period_name, period_complete_day))
             self.store(period_name, period_complete_day, data,)
@@ -115,6 +120,7 @@
             self.sitewide_stats( period_name )
 
             self.update_social_info(period_name, start_date, end_date)
+
 
     def update_social_info(self, period_name, start_date, end_date):
         start_date = start_date.strftime('%Y-%m-%d')
@@ -137,12 +143,11 @@
         data = collections.defaultdict(list)
         rows = results.get('rows',[])
         for row in rows:
-            from ga_model import _normalize_url
             data[_normalize_url(row[0])].append( (row[1], int(row[2]),) )
         ga_model.update_social(period_name, data)
 
 
-    def download(self, start_date, end_date, path='~/dataset/[a-z0-9-_]+'):
+    def download(self, start_date, end_date, path=None):
         '''Get data from GA for a given time period'''
         start_date = start_date.strftime('%Y-%m-%d')
         end_date = end_date.strftime('%Y-%m-%d')
@@ -165,7 +170,10 @@
         packages = []
         for entry in results.get('rows'):
             (loc,pageviews,visits) = entry
-            packages.append( ('http:/' + loc, pageviews, visits,) ) # Temporary hack
+            url = _normalize_url('http:/' + loc)
+            if not url.startswith('/dataset/') and not url.startswith('/publisher/'):
+                continue
+            packages.append( (url, pageviews, visits,) ) # Temporary hack
         return dict(url=packages)
 
     def store(self, period_name, period_complete_day, data):

--- a/ckanext/ga_report/ga_model.py
+++ b/ckanext/ga_report/ga_model.py
@@ -1,10 +1,10 @@
 import re
 import uuid
 
-from sqlalchemy import Table, Column, MetaData
+from sqlalchemy import Table, Column, MetaData, ForeignKey
 from sqlalchemy import types
 from sqlalchemy.sql import select
-from sqlalchemy.orm import mapper
+from sqlalchemy.orm import mapper, relation
 from sqlalchemy import func
 
 import ckan.model as model
@@ -14,8 +14,6 @@
     return unicode(uuid.uuid4())
 
 metadata = MetaData()
-
-
 
 class GA_Url(object):
 
@@ -32,6 +30,7 @@
                       Column('visitors', types.UnicodeText),
                       Column('url', types.UnicodeText),
                       Column('department_id', types.UnicodeText),
+                      Column('package_id', types.UnicodeText),
                 )
 mapper(GA_Url, url_table)
 
@@ -157,34 +156,71 @@
         model.Session.commit()
 
 
+def update_url_stat_totals(period_name):
+
+    """
+        items = model.Session.query(GA_Url).\
+            filter(GA_Url.period_name != "All").\
+            filter(GA_Url.url==url).all()
+        values = {'id': make_uuid(),
+                  'period_name': "All",
+                  'period_complete_day': "0",
+                  'url': url,
+                  'pageviews': sum([int(x.pageviews) for x in items]),
+                  'visitors': sum([int(x.visitors) for x in items]),
+                  'department_id': department_id,
+                  'package_id': package
+                 }
+        model.Session.add(GA_Url(**values))
+        model.Session.commit()
+    """
+
+def pre_update_url_stats(period_name):
+    model.Session.query(GA_Url).\
+            filter(GA_Url.period_name==period_name).delete()
+    model.Session.query(GA_Url).\
+            filter(GA_Url.period_name=='All').delete()
+
 
 def update_url_stats(period_name, period_complete_day, url_data):
+
     for url, views, visitors in url_data:
-        url = _normalize_url(url)
         department_id = _get_department_id_of_url(url)
 
-        # see if the row for this url & month is in the table already
-        item = model.Session.query(GA_Url).\
-            filter(GA_Url.period_name==period_name).\
-            filter(GA_Url.url==url).first()
-        if item:
-            item.period_name = period_name
-            item.pageviews = views
-            item.visitors = visitors
-            item.department_id = department_id
-            model.Session.add(item)
-        else:
-            # create the row
+        package = None
+        if url.startswith('/dataset/'):
+            package = url[len('/dataset/'):]
+
+        values = {'id': make_uuid(),
+                  'period_name': period_name,
+                  'period_complete_day': period_complete_day,
+                  'url': url,
+                  'pageviews': views,
+                  'visitors': visitors,
+                  'department_id': department_id,
+                  'package_id': package
+                 }
+        model.Session.add(GA_Url(**values))
+        model.Session.commit()
+
+        if package:
+            entries = model.Session.query(GA_Url).\
+                filter(GA_Url.period_name!='All').\
+                filter(GA_Url.url==url).all()
             values = {'id': make_uuid(),
-                      'period_name': period_name,
-                      'period_complete_day': period_complete_day,
+                      'period_name': 'All',
+                      'period_complete_day': 0,
                       'url': url,
-                      'pageviews': views,
-                      'visitors': visitors,
-                      'department_id': department_id
+                      'pageviews': sum([int(e.pageviews) for e in entries]),
+                      'visitors': sum([int(e.visitors) for e in entries]),
+                      'department_id': department_id,
+                      'package_id': package
                      }
             model.Session.add(GA_Url(**values))
-        model.Session.commit()
+            model.Session.commit()
+
+
+
 
 
 def update_social(period_name, data):

--- a/ckanext/ga_report/helpers.py
+++ b/ckanext/ga_report/helpers.py
@@ -1,7 +1,9 @@
 import logging
 import operator
+
 import ckan.lib.base as base
 import ckan.model as model
+from ckan.logic import get_action
 
 from ckanext.ga_report.ga_model import GA_Url, GA_Publisher
 from ckanext.ga_report.controller import _get_publishers
@@ -39,25 +41,38 @@
                    order_by('ga_url.pageviews::int desc')
     num_top_datasets = top_datasets.count()
 
+    dataset = None
     if num_top_datasets:
-        dataset = None
+        count = 0
         while not dataset:
             rand = random.randrange(0, min(top, num_top_datasets))
             ga_url = top_datasets[rand]
             dataset = model.Package.get(ga_url.url[len('/dataset/'):])
             if dataset and not dataset.state == 'active':
                 dataset = None
-    else:
+                count += 1
+                if count > 10:
+                    break
+    if not dataset:
+        # fallback
         dataset = model.Session.query(model.Package)\
                   .filter_by(state='active').first()
-    publisher = dataset.get_groups('publisher')[0]
-    return {
-        'dataset': dataset,
-        'publisher': publisher
-    }
+        if not dataset:
+            return None
+    dataset_dict = get_action('package_show')({'model': model,
+                                               'session': model.Session},
+                                              {'id':dataset.id})
+    return dataset_dict
 
 def single_popular_dataset_html(top=20):
-    context = single_popular_dataset(top)
+    dataset_dict = single_popular_dataset(top)
+    groups = package.get('groups', [])
+    publishers = [ g for g in groups if g.get('type') == 'publisher' ]
+    publisher = publishers[0] if publishers else {'name':'', 'title': ''}
+    context = {
+        'dataset': dataset_dict,
+        'publisher': publisher_dict
+        }
     return base.render_snippet('ga_report/ga_popular_single.html', **context)
 
 

--- a/ckanext/ga_report/templates/ga_report/ga_popular_single.html
+++ b/ckanext/ga_report/templates/ga_report/ga_popular_single.html
@@ -8,14 +8,14 @@
     <h2>Featured dataset</h2>
 
     <div class="dataset-summary boxed">
-          <a class="dataset-header" href="${h.url_for(controller='package', action='read', id=dataset.name)}">
-            <h3>${dataset.title}</h3>
+          <a class="dataset-header" href="${h.url_for(controller='package', action='read', id=dataset['name'])}">
+            <h3>${dataset['title']}</h3>
           </a>
           <h4>
             <strong>Publisher</strong> :
-            <a href="/publisher/${publisher.name}">${publisher.title}</a>
+            <a href="/publisher/${publisher['name']}">${publisher['title']}</a>
           </h4>
-          <div>${h.truncate(dataset.notes, length=200, whole_word=True)}</div>
+          <div>${h.truncate(dataset['notes_rendered'], length=200, whole_word=True)}</div>
     </div>
     <div>
       <a href="${h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport',action='read')}" class="btn">Other popular datasets</a>

--- a/ckanext/ga_report/templates/ga_report/publisher/index.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/index.html
@@ -41,15 +41,15 @@
        <table class="table table-condensed table-bordered table-striped">
 	 <tr>
 	   <th>Publisher</th>
+	   <th>Dataset Visits</th>
 	   <th>Dataset Views</th>
-	   <th>Dataset Visits</th>
 	 </tr>
         <py:for each="publisher, views, visits in c.top_publishers">
 	  <tr>
 	    <td>${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name))}
 	    </td>
+	    <td>${visits}</td>
 	    <td>${views}</td>
-	    <td>${visits}</td>
 	  </tr>
         </py:for>
        </table>

--- a/ckanext/ga_report/templates/ga_report/publisher/read.html
+++ b/ckanext/ga_report/templates/ga_report/publisher/read.html
@@ -47,15 +47,15 @@
      <table py:if="c.top_packages" class="table table-condensed table-bordered table-striped">
 	 <tr>
 	   <th>Dataset</th>
+	   <th>Visits</th>
 	   <th>Views</th>
-	   <th>Visits</th>
 	 </tr>
         <py:for each="package, views, visits in c.top_packages">
 	  <tr>
 	    <td>${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}
 	    </td>
+	    <td>${visits}</td>
 	    <td>${views}</td>
-	    <td>${visits}</td>
 	  </tr>
         </py:for>
      </table>