From: Ross Jones Date: Thu, 25 Oct 2012 14:48:13 +0000 Subject: Tidying up X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=fb9969ea4b29984460f9a7a209c98a07481bf4c2 --- Tidying up --- --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ 1. Activate you CKAN python environment and install this extension's software:: $ pyenv/bin/activate - $ pip install -e git+https://github.com/okfn/ckanext-ga-report.git#egg=ckanext-ga-report + $ pip install -e git+https://github.com/datagovuk/ckanext-ga-report.git#egg=ckanext-ga-report 2. Ensure you development.ini (or similar) contains the info about your Google Analytics account and configuration:: --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -66,8 +66,7 @@ And where is: all - data for all time latest - (default) just the 'latest' data - YYYY-MM-DD - just data for all time periods going - back to (and including) this date + YYYY-MM - just data for the specific month """ summary = __doc__.split('\n')[0] usage = __doc__ @@ -96,6 +95,7 @@ elif time_period == 'latest': downloader.latest() else: - since_date = datetime.datetime.strptime(time_period, '%Y-%m-%d') - downloader.since_date(since_date) + # The month to use + for_date = datetime.datetime.strptime(time_period, '%Y-%m') + downloader.specific_month(for_date) --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -1,15 +1,82 @@ import logging import operator -from ckan.lib.base import BaseController, c, render,request +from ckan.lib.base import BaseController, c, render, request, response +import sqlalchemy +from sqlalchemy import func, cast, Integer import ckan.model as model -from ga_model import GA_Url +from ga_model import GA_Url, GA_Stat log = logging.getLogger('ckanext.ga-report') + +def _get_month_name(strdate): + import calendar + from time import strptime + d = strptime(strdate, '%Y-%m') + return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) + + +def _month_details(cls): + months = [] + vals = model.Session.query(cls.period_name).distinct().all() + for m in vals: + months.append( (m[0], _get_month_name(m[0]))) + return sorted(months, key=operator.itemgetter(0), reverse=True) + + class GaReport(BaseController): + def csv(self, month): + import csv + + entries = model.Session.query(GA_Stat).\ + filter(GA_Stat.period_name==month).\ + order_by('GA_Stat.stat_name, GA_Stat.key').all() + + response.headers['Content-Type'] = "text/csv; charset=utf-8" + + writer = csv.writer(response) + writer.writerow(["Period", "Statistic", "Key", "Value"]) + + for entry in entries: + writer.writerow([entry.period_name.encode('utf-8'), + entry.stat_name.encode('utf-8'), + entry.key.encode('utf-8'), + entry.value.encode('utf-8')]) + def index(self): + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Stat) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', c.months[0][0] if c.months else '') + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + entries = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Totals').\ + filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.key').all() + c.global_totals = [(s.key, s.value) for s in entries ] + + keys = { + 'Browser versions': 'browsers', + 'Operating Systems versions': 'os', + 'Social sources': 'social_networks', + 'Languages': 'languages', + 'Country': 'country' + } + + for k, v in keys.iteritems(): + entries = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name==k).\ + filter(GA_Stat.period_name==c.month).\ + order_by('ga_stat.value::int desc').all() + setattr(c, v, [(s.key, s.value) for s in entries ]) + + return render('ga_report/site/index.html') @@ -19,26 +86,40 @@ the datasets associated with the publisher. """ - def _get_month_name(self, str): - import calendar - from time import strptime - d = strptime('2012-10', '%Y-%m') - return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) + def index(self): + + # Get the month details by fetching distinct values and determining the + # month names from the values. + c.months = _month_details(GA_Url) + + # Work out which month to show, based on query params of the first item + c.month = request.params.get('month', c.months[0][0] if c.months else '') + c.month_desc = ''.join([m[1] for m in c.months if m[0]==c.month]) + + connection = model.Session.connection() + q = """ + select department_id, sum(pageviews::int) views, sum(visitors::int) visits + from ga_url + where department_id <> '' + and not url like '/publisher/%%' + and period_name=%s + group by department_id order by views desc limit 20; + """ + c.top_publishers = [] + res = connection.execute(q, c.month) + for row in res: + c.top_publishers.append((model.Group.get(row[0]), row[1], row[2])) + + return render('ga_report/publisher/index.html') - def index(self, id): + def read(self, id): c.publisher = model.Group.get(id) c.top_packages = [] # package, dataset_views in c.top_packages # Get the month details by fetching distinct values and determining the # month names from the values. - c.months = [] - vals = model.Session.query(GA_Url.period_name).distinct().all() - for m in vals: - c.months.append( (m[0],self._get_month_name(m))) - - # Sort the months, so most recent is at the head of our list - c.months = sorted(c.months, key=operator.itemgetter(0), reverse=True) + c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', c.months[0][0] if c.months else '') @@ -52,11 +133,11 @@ entries = model.Session.query(GA_Url).\ filter(GA_Url.department_id==c.publisher.name).\ filter(GA_Url.period_name==c.month).\ - order_by('ga_url.pageviews desc').all() + order_by('ga_url.pageviews::int desc')[:20] for entry in entries: if entry.url.startswith('/dataset/'): p = model.Package.get(entry.url[len('/dataset/'):]) c.top_packages.append((p,entry.pageviews,entry.visitors)) - return render('ga_report/publisher/index.html') + return render('ga_report/publisher/read.html') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -21,8 +21,17 @@ self.profile_id = profile_id - def all_(self): - self.since_date(datetime.datetime(2010, 1, 1)) + def specific_month(self, date): + import calendar + + first_of_this_month = datetime.datetime(date.year, date.month, 1) + _, last_day_of_month = calendar.monthrange(int(date.year), int(date.month)) + last_of_this_month = datetime.datetime(date.year, date.month, last_day_of_month) + periods = ((date.strftime(FORMAT_MONTH), + last_day_of_month, + first_of_this_month, last_of_this_month),) + self.download_and_store(periods) + def latest(self): if self.period == 'monthly': @@ -37,13 +46,13 @@ self.download_and_store(periods) - def since_date(self, since_date): + def for_date(self, for_date): assert isinstance(since_date, datetime.datetime) periods = [] # (period_name, period_complete_day, start_date, end_date) if self.period == 'monthly': first_of_the_months_until_now = [] - year = since_date.year - month = since_date.month + year = for_date.year + month = for_date.month now = datetime.datetime.now() first_of_this_month = datetime.datetime(now.year, now.month, 1) while True: @@ -85,7 +94,7 @@ self.get_full_period_name(period_name, period_complete_day), start_date.strftime('%Y %m %d'), end_date.strftime('%Y %m %d')) - """ + data = self.download(start_date, end_date, '~/dataset/[a-z0-9-_]+') log.info('Storing Dataset Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) @@ -95,7 +104,7 @@ log.info('Storing Publisher Analytics for period "%s"', self.get_full_period_name(period_name, period_complete_day)) self.store(period_name, period_complete_day, data,) - """ + ga_model.update_publisher_stats(period_name) # about 30 seconds. self.sitewide_stats( period_name ) --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -18,14 +18,24 @@ def after_map(self, map): map.connect( - '/data/analytics/', - controller='ckanext.ga_report.controller:GaReport', + '/data/analytics/publisher', + controller='ckanext.ga_report.controller:GaPublisherReport', action='index' ) map.connect( '/data/analytics/publisher/{id}', controller='ckanext.ga_report.controller:GaPublisherReport', + action='read' + ) + map.connect( + '/data/analytics', + controller='ckanext.ga_report.controller:GaReport', action='index' + ) + map.connect( + '/data/analytics/data_{month}.csv', + controller='ckanext.ga_report.controller:GaReport', + action='csv' ) return map --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -3,15 +3,19 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> - Analytics for ${g.site_title} + Publisher Analytics for ${g.site_title} + + +
  • +

    Publishers

    +
    • +
    -

    Analytics for ${c.publisher.title}

    +

    Publisher Analytics

    +

    The top 20 publishers

    -

    Most viewed datasets

    -

    Note: this data does not include API calls

    - -
    +
    + + + + + +
    +
    + + + + + + + + + + + + + + +
    DatasetViewsVisits
    ${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} + ${views}${visits}
    + + +
    + + + + + + + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -1,1 +1,147 @@ -HAI Site + + + Site analytics + + +
  • +

    Statistics

    +

    It is possible to export the analytics data as a CSV file, which contains all of the information for ${c.month_desc}

    + +
    • +     + +
    +

    Site statistics

    + +
    +
    + + +
    +
    + +
    + +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    +
    + + + + + + + + + + + +
    NameValue
    ${name}${value}
    +
    + + +
    +
    + + + +
    + + + + + + + + + + + --- a/ckanext/ga_report/tests/test_api.py +++ b/ckanext/ga_report/tests/test_api.py @@ -32,7 +32,7 @@ svc = init_service("token.dat", "credentials.json") downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) try: - downloader.since_date(datetime.datetime.now() - datetime.timedelta(days=-30)) + downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30)) except Exception as e: assert False, e --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ [paste.paster_command] loadanalytics = ckanext.ga_report.command:LoadAnalytics initdb = ckanext.ga_report.command:InitDB + getauthtoken = ckanext.ga_report.command:GetAuthToken """, )