From: Ross Jones Date: Tue, 30 Oct 2012 14:57:39 +0000 Subject: Fixes to show All Time data X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=bc5b389806da2a721ae1b8fb2d873a5a190dfdaa --- Fixes to show All Time data --- --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ 1. Activate you CKAN python environment and install this extension's software:: $ pyenv/bin/activate - $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report + $ pip install -e git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration:: --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -66,8 +66,7 @@ And where is: all - data for all time latest - (default) just the 'latest' data - YYYY-MM-DD - just data for all time periods going - back to (and including) this date + YYYY-MM - just data for the specific month """ summary = __doc__.split('\n')[0] usage = __doc__ @@ -96,6 +95,7 @@ elif time_period == 'latest': downloader.latest() else: - since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d') - downloader.since_date(since_date) + # The month to use + for_date = datetime.datetime.strptime(time_period, '%Y-%m') + downloader.specific_month(for_date) --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,6 +1,6 @@ import logging import operator -from ckan.lib.base import BaseController, c, render, request, response +from ckan.lib.base import BaseController, c, render, request, response, abort import sqlalchemy from sqlalchemy import func, cast, Integer @@ -10,10 +10,10 @@ log = logging.getLogger('ckanext.ga-report') -def _get_month_name(str): +def _get_month_name(strdate): import calendar from time import strptime - d = strptime('2012-10', '%Y-%m') + d = strptime(strdate, '%Y-%m') return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) @@ -21,7 +21,7 @@ months = [] vals = model.Session.query(cls.period_name).distinct().all() for m in vals: - months.append( (m[0], _get_month_name(m))) + months.append( (m[0], _get_month_name(m[0]))) return sorted(months, key=operator.itemgetter(0), reverse=True) @@ -34,7 +34,6 @@ filter(GA_Stat.period_name==month).\ order_by('GA_Stat.stat_name, GA_Stat.key').all() - response.headers['Content-disposition'] = 'attachment; filename=dgu_analytics_%s.csv' % (month) response.headers['Content-Type'] = "text/csv; charset=utf-8" writer = csv.writer(response) @@ -58,8 +57,21 @@ entries = model.Session.query(GA_Stat).\ filter(GA_Stat.stat_name=='Totals').\ - filter(GA_Stat.period_name==c.month).all() - c.global_totals = [(s.key, s.value) for s in entries ] + filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.key').all() + + c.global_totals = [] + for e in entries: + val = e.value + if e.key in ['Average time on site', 'Pages per visit', 'Percent new visits']: + val = "%.2f" % round(float(e.value), 2) + if e.key == 'Average time on site': + mins, secs = divmod(float(val), 60) + hours, mins = divmod(mins, 60) + val = '%02d:%02d:%02d (%s seconds) ' % (hours, mins, secs, val) + e.key = '%s *' % e.key + c.global_totals.append((e.key, val)) + keys = { 'Browser versions': 'browsers', @@ -87,25 +99,37 @@ """ def index(self): + # Get the month details by fetching distinct values and determining the # month names from the values. c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + c.month = request.params.get('month', '') + c.month_desc = 'all time' + if c.month: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) connection = model.Session.connection() q = """ select department_id, sum(pageviews::int) views, sum(visitors::int) visits from ga_url - where department_id <> '' - and not url like '/publisher/%%' - and period_name=%s - group by department_id order by views desc limit 20; - """ + where department_id <> ''""" + if c.month: + q = q + """ + and period_name=%s + """ + q = q + """ + group by department_id order by views desc limit 20; + """ + + # Add this back (before and period_name =%s) if you want to ignore publisher + # homepage views + # and not url like '/publisher/%%' + c.top_publishers = [] res = connection.execute(q, c.month) + for row in res: c.top_publishers.append((model.Group.get(row[0]), row[1], row[2])) @@ -113,7 +137,11 @@ def read(self, id): + count = 20 + c.publisher = model.Group.get(id) + if not c.publisher: + abort(404, 'A publisher with that name could not be found') c.top_packages = [] # package, dataset_views in c.top_packages # Get the month details by fetching distinct values and determining the @@ -121,22 +149,50 @@ c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item - c.month = request.params.get('month', c.months[0][0] if c.months else '') - c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + c.month = request.params.get('month', '') + if not c.month: + c.month_desc = 'all time' + else: + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) - entry = model.Session.query(GA_Url).\ - filter(GA_Url.url=='/publisher/%s' % c.publisher.name).\ - filter(GA_Url.period_name==c.month).first() - c.publisher_page_views = entry.pageviews if entry else 0 + c.publisher_page_views = 0 + q = model.Session.query(GA_Url).\ + filter(GA_Url.url=='/publisher/%s' % c.publisher.name) + if c.month: + entry = q.filter(GA_Url.period_name==c.month).first() + c.publisher_page_views = entry.pageviews if entry else 0 + else: + for e in q.all(): + c.publisher_page_views = c.publisher_page_views + int(e.pageviews) - entries = model.Session.query(GA_Url).\ + + q = model.Session.query(GA_Url).\ filter(GA_Url.department_id==c.publisher.name).\ - filter(GA_Url.period_name==c.month).\ - order_by('ga_url.pageviews::int desc')[:20] - for entry in entries: - if entry.url.startswith('/dataset/'): + filter(GA_Url.url.like('/dataset/%')) + if c.month: + q = q.filter(GA_Url.period_name==c.month) + q = q.order_by('ga_url.pageviews::int desc') + + if c.month: + for entry in q[:count]: p = model.Package.get(entry.url[len('/dataset/'):]) c.top_packages.append((p,entry.pageviews,entry.visitors)) + else: + ds = {} + for entry in q.all(): + if len(ds) >= count: + break + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in ds: + ds[p] = {'views':0, 'visits': 0} + ds[p]['views'] = ds[p]['views'] + int(entry.pageviews) + ds[p]['visits'] = ds[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in ds.iteritems(): + results.append((k,v['views'],v['visits'])) + + c.top_packages = sorted(results, key=operator.itemgetter(1), reverse=True) return render('ga_report/publisher/read.html') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -21,8 +21,17 @@ self.profile_id = profile_id - def all_(self): - self.since_date(datetime.datetime(2010, 1, 1)) + def specific_month(self, date): + import calendar + + first_of_this_month = datetime.datetime(date.year, date.month, 1) + _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month)) + last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month) + periods = ((date.strftime(FORMAT_MONTH), + last_day_of_month, + first_of_this_month, last_of_this_month),) + self.download_and_store(periods) + def latest(self): if self.period == 'monthly': @@ -37,13 +46,13 @@ self.download_and_store(periods) - def since_date(self, since_date): + def for_date(self, for_date): assert isinstance(since_date, datetime.datetime) periods = [] # (period_name, period_complete_day, start_date, end_date) if self.period == 'monthly': first_of_the_months_until_now = [] - year = since_date.year - month = since_date.month + year = for_date.year + month = for_date.month now = datetime.datetime.now() first_of_this_month = datetime.datetime(now.year, now.month, 1) while True: @@ -85,7 +94,7 @@ self.get_full_period_name(period_name, period_complete_day), start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - """ + data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) @@ -95,7 +104,7 @@ log.info('Storing Publisher Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) self.store(period_name, period_complete_day, data,) - """ + ga_model.update_publisher_stats(period_name) # about 30 seconds. self.sitewide_stats( period_name ) --- /dev/null +++ b/ckanext/ga_report/helpers.py @@ -1,1 +1,42 @@ +import logging +import operator +import ckan.lib.base as base +import ckan.model as model +_log = logging.getLogger(__name__) + +def most_popular_datasets(publisher, count=20): + from ckanext.ga_report.ga_model import GA_Url + + if not publisher: + _log.error("No valid publisher passed to 'most_popular_datasets'") + return "" + + datasets = {} + entries = model.Session.query(GA_Url).\ + filter(GA_Url.department_id==publisher.name).\ + filter(GA_Url.url.like('/dataset/%')).\ + order_by('ga_url.pageviews::int desc').all() + for entry in entries: + if len(datasets) < count: + p = model.Package.get(entry.url[len('/dataset/'):]) + if not p in datasets: + datasets[p] = {'views':0, 'visits': 0} + datasets[p]['views'] = datasets[p]['views'] + int(entry.pageviews) + datasets[p]['visits'] = datasets[p]['visits'] + int(entry.visitors) + + results = [] + for k, v in datasets.iteritems(): + results.append((k,v['views'],v['visits'])) + + results = sorted(results, key=operator.itemgetter(1), reverse=True) + + ctx = { + 'dataset_count': len(datasets), + 'datasets': results, + + 'publisher': publisher + } + + return base.render_snippet('ga_report/publisher/popular.html', **ctx) + --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -2,33 +2,32 @@ import ckan.lib.helpers as h import ckan.plugins as p from ckan.plugins import implements, toolkit -#import gasnippet -#import commands -#import dbutil log = logging.getLogger('ckanext.ga-report') class GAReportPlugin(p.SingletonPlugin): implements(p.IConfigurer, inherit=True) implements(p.IRoutes, inherit=True) + implements(p.ITemplateHelpers, inherit=True) def update_config(self, config): toolkit.add_template_directory(config, 'templates') toolkit.add_public_directory(config, 'public') + def get_helpers(self): + """ + A dictionary of extra helpers that will be available to provide + ga report info to templates. + """ + from ckanext.ga_report.helpers import most_popular_datasets + return { + 'ga_report_installed': lambda: True, + 'most_popular_datasets': most_popular_datasets, + } + def after_map(self, map): map.connect( - '/data/analytics', - controller='ckanext.ga_report.controller:GaReport', - action='index' - ) - map.connect( - '/data/analytics_{month}.csv', - controller='ckanext.ga_report.controller:GaReport', - action='csv' - ) - map.connect( - '/data/analytics/publisher/', + '/data/analytics/publisher', controller='ckanext.ga_report.controller:GaPublisherReport', action='index' ) @@ -37,6 +36,16 @@ controller='ckanext.ga_report.controller:GaPublisherReport', action='read' ) + map.connect( + '/data/analytics', + controller='ckanext.ga_report.controller:GaReport', + action='index' + ) + map.connect( + '/data/analytics/data_{month}.csv', + controller='ckanext.ga_report.controller:GaReport', + action='csv' + ) return map --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -8,16 +8,19 @@

  • Publishers

    +

    The table shows the top 20 publishers as recorded by page views of datasets owned by that publisher, and the number of visits to each publisher's home page.

    •

    Publisher Analytics

    -

    The top 20 publishers

    +

    The top 20 publishers of ${c.month_desc}

    + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -8,6 +8,12 @@

  • Statistics

    +

    It is possible to export the analytics data as a CSV file, which contains all of the information for ${c.month_desc}

    +
    • +
  • +

    Publisher statistics

    +

    You can view statistics about specific publishers at the Publisher Analytics page

    +
    • @@ -24,8 +30,6 @@
    - -

    Export data as CSV

      @@ -50,6 +54,8 @@ + +

      * Values are rounded up to 2 decimal places.

    --- a/ckanext/ga_report/tests/test_api.py +++ b/ckanext/ga_report/tests/test_api.py @@ -32,7 +32,7 @@ svc = init_service("token.dat", "credentials.json") downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) try: - downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30)) + downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30)) except Exception as e: assert False, e --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ [paste.paster_command] loadanalytics = ckanext.ga_report.command:LoadAnalytics initdb = ckanext.ga_report.command:InitDB + getauthtoken = ckanext.ga_report.command:GetAuthToken """, )