Added support for a popular datasets widget.
[ckanext-ga-report.git] / ckanext / ga_report / controller.py
blob:a/ckanext/ga_report/controller.py -> blob:b/ckanext/ga_report/controller.py
--- a/ckanext/ga_report/controller.py
+++ b/ckanext/ga_report/controller.py
@@ -9,7 +9,7 @@
 import sqlalchemy
 from sqlalchemy import func, cast, Integer
 import ckan.model as model
-from ga_model import GA_Url, GA_Stat, GA_ReferralStat
+from ga_model import GA_Url, GA_Stat, GA_ReferralStat, GA_Publisher
 
 log = logging.getLogger('ckanext.ga-report')
 
@@ -22,8 +22,9 @@
 
 
 def _month_details(cls):
+    '''Returns a list of all the month names'''
     months = []
-    vals = model.Session.query(cls.period_name).distinct().all()
+    vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all()
     for m in vals:
         months.append( (m[0], _get_month_name(m[0])))
     return sorted(months, key=operator.itemgetter(0), reverse=True)
@@ -70,15 +71,15 @@
         entries = q.order_by('ga_stat.key').all()
 
         def clean_key(key, val):
-            if key in ['Average time on site', 'Pages per visit', 'New visits']:
+            if key in ['Average time on site', 'Pages per visit', 'New visits', 'Bounce rate (home page)']:
                 val =  "%.2f" % round(float(val), 2)
                 if key == 'Average time on site':
                     mins, secs = divmod(float(val), 60)
                     hours, mins = divmod(mins, 60)
                     val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val)
-                if key == 'New visits':
+                if key in ['New visits','Bounce rate (home page)']:
                     val = "%s%%" % val
-            if key in ['Bounces', 'Total page views', 'Total visits']:
+            if key in ['Total page views', 'Total visits']:
                 val = int(val)
 
             return key, val
@@ -93,11 +94,12 @@
             for e in entries:
                 d[e.key].append(float(e.value))
             for k, v in d.iteritems():
-                if k in ['Bounces', 'Total page views', 'Total visits']:
+                if k in ['Total page views', 'Total visits']:
                     v = sum(v)
                 else:
                     v = float(sum(v))/len(v)
                 key, val = clean_key(k,v)
+
                 c.global_totals.append((key, val))
                 c.global_totals = sorted(c.global_totals, key=operator.itemgetter(0))
 
@@ -134,29 +136,7 @@
             c.social_referrer_totals.append((shorten_name(entry[0]), fill_out_url(entry[0]),'',
                                             entry[1]))
 
-
-        browser_version_re = re.compile("(.*)\((.*)\)")
         for k, v in keys.iteritems():
-
-            def clean_field(key):
-                if k != 'Browser versions':
-                    return key
-                m = browser_version_re.match(key)
-                browser = m.groups()[0].strip()
-                ver = m.groups()[1]
-                parts = ver.split('.')
-                if len(parts) > 1:
-                    if parts[1][0] == '0':
-                        ver = parts[0]
-                    else:
-                        ver = "%s.%s" % (parts[0],parts[1])
-                if browser in ['Safari','Android Browser']:  # Special case complex version nums
-                    ver = parts[0]
-                    if len(ver) > 2:
-                        ver = "%s%sX" % (ver[0], ver[1])
-
-                return "%s (%s)" % (browser, ver,)
-
             q = model.Session.query(GA_Stat).\
                 filter(GA_Stat.stat_name==k)
             if c.month:
@@ -166,15 +146,11 @@
 
             d = collections.defaultdict(int)
             for e in q.all():
-                d[clean_field(e.key)] += int(e.value)
+                d[e.key] += int(e.value)
             entries = []
             for key, val in d.iteritems():
                 entries.append((key,val,))
             entries = sorted(entries, key=operator.itemgetter(1), reverse=True)
-
-            def percent(num, total):
-                p = 100 * float(num)/float(total)
-                return "%.2f%%" % round(p, 2)
 
             # Get the total for each set of values and then set the value as
             # a percentage of the total
@@ -182,57 +158,65 @@
                 total = sum([x for n,x in c.global_totals if n == 'Total visits'])
             else:
                 total = sum([num for _,num in entries])
-            setattr(c, v, [(k,percent(v,total)) for k,v in entries ])
+            setattr(c, v, [(k,_percent(v,total)) for k,v in entries ])
 
         return render('ga_report/site/index.html')
 
 
-class GaPublisherReport(BaseController):
+class GaDatasetReport(BaseController):
     """
-    Displays the pageview and visit count for specific publishers based on
-    the datasets associated with the publisher.
+    Displays the pageview and visit count for datasets
+    with options to filter by publisher and time period.
     """
-    def csv(self, month):
-
-        c.month = month if not month =='all' else ''
+    def publisher_csv(self, month):
+        '''
+        Returns a CSV of each publisher with the total number of dataset
+        views & visits.
+        '''
+        c.month = month if not month == 'all' else ''
         response.headers['Content-Type'] = "text/csv; charset=utf-8"
         response.headers['Content-Disposition'] = str('attachment; filename=publishers_%s.csv' % (month,))
 
         writer = csv.writer(response)
-        writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
-
-        for publisher,view,visit in _get_publishers(None):
+        writer.writerow(["Publisher Title", "Publisher Name", "Views", "Visits", "Period Name"])
+
+        for publisher,view,visit in _get_top_publishers(None):
             writer.writerow([publisher.title.encode('utf-8'),
+                             publisher.name.encode('utf-8'),
                              view,
                              visit,
                              month])
 
-
-
-    def publisher_csv(self, id, month):
-
-        c.month = month if not month =='all' else ''
-        c.publisher = model.Group.get(id)
-        if not c.publisher:
-            abort(404, 'A publisher with that name could not be found')
+    def dataset_csv(self, id='all', month='all'):
+        '''
+        Returns a CSV with the number of views & visits for each dataset.
+
+        :param id: A Publisher ID or None if you want for all
+        :param month: The time period, or 'all'
+        '''
+        c.month = month if not month == 'all' else ''
+        if id != 'all':
+            c.publisher = model.Group.get(id)
+            if not c.publisher:
+                abort(404, 'A publisher with that name could not be found')
 
         packages = self._get_packages(c.publisher)
         response.headers['Content-Type'] = "text/csv; charset=utf-8"
         response.headers['Content-Disposition'] = \
-            str('attachment; filename=%s_%s.csv' % (c.publisher.name, month,))
+            str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,))
 
         writer = csv.writer(response)
-        writer.writerow(["Publisher", "Views", "Visits", "Period Name"])
+        writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Period Name"])
 
         for package,view,visit in packages:
             writer.writerow([package.title.encode('utf-8'),
+                             package.name.encode('utf-8'),
                              view,
                              visit,
                              month])
 
-
-
-    def index(self):
+    def publishers(self):
+        '''A list of publishers and the number of views/visits for each'''
 
         # Get the month details by fetching distinct values and determining the
         # month names from the values.
@@ -244,52 +228,52 @@
         if c.month:
             c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
 
-        c.top_publishers = _get_publishers()
-
+        c.top_publishers = _get_top_publishers()
         return render('ga_report/publisher/index.html')
 
-
-    def _get_packages(self, publisher, count=-1):
+    def _get_packages(self, publisher=None, count=-1):
+        '''Returns the datasets in order of visits'''
         if count == -1:
             count = sys.maxint
 
+        month = c.month or 'All'
+
+        q = model.Session.query(GA_Url,model.Package)\
+            .filter(model.Package.name==GA_Url.package_id)\
+            .filter(GA_Url.url.like('/dataset/%'))
+        if publisher:
+            q = q.filter(GA_Url.department_id==publisher.name)
+        q = q.filter(GA_Url.period_name==month)
+        q = q.order_by('ga_url.visits::int desc')
         top_packages = []
-        q =  model.Session.query(GA_Url).\
-            filter(GA_Url.department_id==publisher.name).\
-            filter(GA_Url.url.like('/dataset/%'))
-        if c.month:
-            q = q.filter(GA_Url.period_name==c.month)
-        q = q.order_by('ga_url.pageviews::int desc')
-
-        if c.month:
-            for entry in q[:count]:
-                p = model.Package.get(entry.url[len('/dataset/'):])
-                top_packages.append((p,entry.pageviews,entry.visitors))
-        else:
-            ds = {}
-            for entry in q.all():
-                if len(ds) >= count:
-                    break
-                p = model.Package.get(entry.url[len('/dataset/'):])
-                if not p in ds:
-                    ds[p] = {'views':0, 'visits': 0}
-                ds[p]['views'] = ds[p]['views'] + int(entry.pageviews)
-                ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors)
-
-            results = []
-            for k, v in ds.iteritems():
-                results.append((k,v['views'],v['visits']))
-
-            top_packages = sorted(results, key=operator.itemgetter(1), reverse=True)
+        for entry,package in q.limit(count):
+            if package:
+                top_packages.append((package, entry.pageviews, entry.visits))
+            else:
+                log.warning('Could not find package associated package')
+
         return top_packages
 
-
-    def read(self, id):
+    def read(self):
+        '''
+        Lists the most popular datasets across all publishers
+        '''
+        return self.read_publisher(None)
+
+    def read_publisher(self, id):
+        '''
+        Lists the most popular datasets for a publisher (or across all publishers)
+        '''
         count = 20
 
-        c.publisher = model.Group.get(id)
-        if not c.publisher:
-            abort(404, 'A publisher with that name could not be found')
+        c.publishers = _get_publishers()
+
+        id = request.params.get('publisher', id)
+        if id and id != 'all':
+            c.publisher = model.Group.get(id)
+            if not c.publisher:
+                abort(404, 'A publisher with that name could not be found')
+            c.publisher_name = c.publisher.name
         c.top_packages = [] # package, dataset_views in c.top_packages
 
         # Get the month details by fetching distinct values and determining the
@@ -303,46 +287,59 @@
         else:
             c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month])
 
+        month = c.month or 'All'
         c.publisher_page_views = 0
         q = model.Session.query(GA_Url).\
-            filter(GA_Url.url=='/publisher/%s' % c.publisher.name)
-        if c.month:
-            entry = q.filter(GA_Url.period_name==c.month).first()
-            c.publisher_page_views = entry.pageviews if entry else 0
-        else:
-            for e in q.all():
-                c.publisher_page_views = c.publisher_page_views  + int(e.pageviews)
+            filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
+        entry = q.filter(GA_Url.period_name==c.month).first()
+        c.publisher_page_views = entry.pageviews if entry else 0
 
         c.top_packages = self._get_packages(c.publisher, 20)
 
         return render('ga_report/publisher/read.html')
 
-def _get_publishers(limit=20):
+def _get_top_publishers(limit=20):
+    '''
+    Returns a list of the top 20 publishers by dataset visits.
+    (The number to show can be varied with 'limit')
+    '''
+    month = c.month or 'All'
     connection = model.Session.connection()
     q = """
-        select department_id, sum(pageviews::int) views, sum(visitors::int) visits
+        select department_id, sum(pageviews::int) views, sum(visits::int) visits
         from ga_url
-        where department_id <> ''"""
-    if c.month:
-        q = q + """
-                and period_name=%s
-        """
-    q = q + """
-            group by department_id order by views desc
+        where department_id <> ''
+          and package_id <> ''
+          and url like '/dataset/%%'
+          and period_name=%s
+        group by department_id order by views desc
         """
     if limit:
         q = q + " limit %s;" % (limit)
 
-    # Add this back (before and period_name =%s) if you want to ignore publisher
-    # homepage views
-    # and not url like '/publisher/%%'
-
     top_publishers = []
-    res = connection.execute(q, c.month)
-
+    res = connection.execute(q, month)
     for row in res:
         g = model.Group.get(row[0])
         if g:
             top_publishers.append((g, row[1], row[2]))
     return top_publishers
 
+
+def _get_publishers():
+    '''
+    Returns a list of all publishers. Each item is a tuple:
+      (names, title)
+    '''
+    publishers = []
+    for pub in model.Session.query(model.Group).\
+               filter(model.Group.type=='publisher').\
+               filter(model.Group.state=='active').\
+               order_by(model.Group.name):
+        publishers.append((pub.name, pub.title))
+    return publishers
+
+def _percent(num, total):
+    p = 100 * float(num)/float(total)
+    return "%.2f%%" % round(p, 2)
+