From: Ross Jones Date: Fri, 11 Jan 2013 14:52:07 +0000 Subject: Implements downloads counts (for dataset resources) and fixes an issue with 'All' records. X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=fb4307a6de326e9aa113465bcabe674fd68d8775 --- Implements downloads counts (for dataset resources) and fixes an issue with 'All' records. - Fetches the data for downloads (either direct or cached) from when a user clicked on a Download button. We lookup the resource for this url (based on .url or .cache_url) and then associate the download count with the package it belongs to. - Fixes a bug (#211) where the All records are deleted for every url, even if that url was not fetched (and therefore won't get a new All record). --- --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.py[co] *.py~ .gitignore +ckan.log # Packages *.egg --- a/ckanext/ga_report/command.py +++ b/ckanext/ga_report/command.py @@ -55,6 +55,36 @@ init_service('token.dat', self.args[0] if self.args else 'credentials.json') + +class FixTimePeriods(CkanCommand): + """ + Fixes the 'All' records for GA_Urls + + It is possible that older urls that haven't recently been visited + do not have All records. This command will traverse through those + records and generate valid All records for them. + """ + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = 0 + min_args = 0 + + def __init__(self, name): + super(FixTimePeriods, self).__init__(name) + + def command(self): + import ckan.model as model + from ga_model import post_update_url_stats + self._load_config() + model.Session.remove() + model.Session.configure(bind=model.meta.engine) + + log = logging.getLogger('ckanext.ga_report') + + log.info("Updating 'All' records for old URLs") + post_update_url_stats() + log.info("Processing complete") + class LoadAnalytics(CkanCommand): --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -13,6 +13,7 @@ log = logging.getLogger('ckanext.ga-report') +DOWNLOADS_AVAILABLE_FROM = '2012-12' def _get_month_name(strdate): import calendar @@ -21,13 +22,34 @@ return '%s %s' % (calendar.month_name[d.tm_mon], d.tm_year) -def _month_details(cls): - '''Returns a list of all the month names''' +def _month_details(cls, stat_key=None): + ''' + Returns a list of all the periods for which we have data, unfortunately + knows too much about the type of the cls being passed as GA_Url has a + more complex query + + This may need extending if we add a period_name to the stats + ''' months = [] - vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all() + day = None + + q = model.Session.query(cls.period_name,cls.period_complete_day)\ + .filter(cls.period_name!='All').distinct(cls.period_name) + if stat_key: + q= q.filter(cls.stat_name==stat_key) + + vals = q.order_by("period_name desc").all() + + if vals and vals[0][1]: + day = int(vals[0][1]) + ordinal = 'th' if 11 <= day <= 13 \ + else {1:'st',2:'nd',3:'rd'}.get(day % 10, 'th') + day = "{day}{ordinal}".format(day=day, ordinal=ordinal) + for m in vals: months.append( (m[0], _get_month_name(m[0]))) - return sorted(months, key=operator.itemgetter(0), reverse=True) + + return months, day class GaReport(BaseController): @@ -35,7 +57,7 @@ def csv(self, month): import csv - q = model.Session.query(GA_Stat) + q = model.Session.query(GA_Stat).filter(GA_Stat.stat_name!='Downloads') if month != 'all': q = q.filter(GA_Stat.period_name==month) entries = q.order_by('GA_Stat.period_name, GA_Stat.stat_name, GA_Stat.key').all() @@ -52,11 +74,12 @@ entry.key.encode('utf-8'), entry.value.encode('utf-8')]) + def index(self): # Get the month details by fetching distinct values and determining the # month names from the values. - c.months = _month_details(GA_Stat) + c.months, c.day = _month_details(GA_Stat) # Work out which month to show, based on query params of the first item c.month_desc = 'all months' @@ -97,7 +120,7 @@ if k in ['Total page views', 'Total visits']: v = sum(v) else: - v = float(sum(v))/len(v) + v = float(sum(v))/float(len(v)) key, val = clean_key(k,v) c.global_totals.append((key, val)) @@ -206,13 +229,14 @@ str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,)) writer = csv.writer(response) - writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Period Name"]) - - for package,view,visit in packages: + writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"]) + + for package,view,visit,downloads in packages: writer.writerow([package.title.encode('utf-8'), package.name.encode('utf-8'), view, visit, + downloads, month]) def publishers(self): @@ -220,7 +244,7 @@ # Get the month details by fetching distinct values and determining the # month names from the values. - c.months = _month_details(GA_Url) + c.months, c.day = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', '') @@ -233,10 +257,10 @@ def _get_packages(self, publisher=None, count=-1): '''Returns the datasets in order of views''' - if count == -1: - count = sys.maxint - + have_download_data = True month = c.month or 'All' + if month != 'All': + have_download_data = month >= DOWNLOADS_AVAILABLE_FROM q = model.Session.query(GA_Url,model.Package)\ .filter(model.Package.name==GA_Url.package_id)\ @@ -246,9 +270,25 @@ q = q.filter(GA_Url.period_name==month) q = q.order_by('ga_url.pageviews::int desc') top_packages = [] - for entry,package in q.limit(count): + if count == -1: + entries = q.all() + else: + entries = q.limit(count) + + for entry,package in entries: if package: - top_packages.append((package, entry.pageviews, entry.visits)) + # Downloads .... + if have_download_data: + dls = model.Session.query(GA_Stat).\ + filter(GA_Stat.stat_name=='Downloads').\ + filter(GA_Stat.key==package.name) + if month != 'All': # Fetch everything unless the month is specific + dls = dls.filter(GA_Stat.period_name==month) + + downloads = sum(int(d.value) for d in dls.all()) + else: + downloads = 'No data' + top_packages.append((package, entry.pageviews, entry.visits, downloads)) else: log.warning('Could not find package associated package') @@ -278,7 +318,7 @@ # Get the month details by fetching distinct values and determining the # month names from the values. - c.months = _month_details(GA_Url) + c.months, c.day = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', '') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -13,6 +13,7 @@ FORMAT_MONTH = '%Y-%m' MIN_VIEWS = 50 MIN_VISITS = 20 +MIN_DOWNLOADS = 10 class DownloadAnalytics(object): '''Downloads and stores analytics info''' @@ -122,11 +123,15 @@ log.info('Storing publisher views (%i rows)', len(data.get('url'))) self.store(period_name, period_complete_day, data,) + # Make sure the All records are correct. + ga_model.post_update_url_stats() + log.info('Aggregating datasets by publisher') ga_model.update_publisher_stats(period_name) # about 30 seconds. + log.info('Downloading and storing analytics for site-wide stats') - self.sitewide_stats( period_name ) + self.sitewide_stats( period_name, period_complete_day ) log.info('Downloading and storing analytics for social networks') self.update_social_info(period_name, start_date, end_date) @@ -153,7 +158,8 @@ data = collections.defaultdict(list) rows = results.get('rows',[]) for row in rows: - data[_normalize_url(row[0])].append( (row[1], int(row[2]),) ) + url = _normalize_url('http:/' + row[0]) + data[url].append( (row[1], int(row[2]),) ) ga_model.update_social(period_name, data) @@ -178,6 +184,7 @@ end_date=end_date).execute() packages = [] + log.info("There are %d results" % results['totalResults']) for entry in results.get('rows'): (loc,pageviews,visits) = entry url = _normalize_url('http:/' + loc) # strips off domain e.g. www.data.gov.uk or data.gov.uk @@ -194,7 +201,7 @@ if 'url' in data: ga_model.update_url_stats(period_name, period_complete_day, data['url']) - def sitewide_stats(self, period_name): + def sitewide_stats(self, period_name, period_complete_day): import calendar year, month = period_name.split('-') _, last_day_of_month = calendar.monthrange(int(year), int(month)) @@ -202,10 +209,10 @@ start_date = '%s-01' % period_name end_date = '%s-%s' % (period_name, last_day_of_month) funcs = ['_totals_stats', '_social_stats', '_os_stats', - '_locale_stats', '_browser_stats', '_mobile_stats'] + '_locale_stats', '_browser_stats', '_mobile_stats', '_download_stats'] for f in funcs: log.info('Downloading analytics for %s' % f.split('_')[1]) - getattr(self, f)(start_date, end_date, period_name) + getattr(self, f)(start_date, end_date, period_name, period_complete_day) def _get_results(result_data, f): data = {} @@ -214,7 +221,7 @@ data[key] = data.get(key,0) + result[1] return data - def _totals_stats(self, start_date, end_date, period_name): + def _totals_stats(self, start_date, end_date, period_name, period_complete_day): """ Fetches distinct totals, total pageviews etc """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -224,7 +231,8 @@ max_results=10000, end_date=end_date).execute() result_data = results.get('rows') - ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}) + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}, + period_complete_day) results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -239,7 +247,7 @@ 'New visits': result_data[0][2], 'Total visits': result_data[0][3], } - ga_model.update_sitewide_stats(period_name, "Totals", data) + ga_model.update_sitewide_stats(period_name, "Totals", data, period_complete_day) # Bounces from / or another configurable page. path = '/%s%s' % (config.get('googleanalytics.account'), @@ -248,7 +256,7 @@ ids='ga:' + self.profile_id, filters='ga:pagePath==%s' % (path,), start_date=start_date, - metrics='ga:bounces,ga:pageviews', + metrics='ga:visitBounceRate', dimensions='ga:pagePath', max_results=10000, end_date=end_date).execute() @@ -258,13 +266,14 @@ path, result_data) return results = result_data[0] - bounces, total = [float(x) for x in result_data[0][1:]] - pct = 100 * bounces/total - log.info('%d bounces from %d total == %s', bounces, total, pct) - ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': pct}) - - - def _locale_stats(self, start_date, end_date, period_name): + bounces = float(results[1]) + # visitBounceRate is already a % + log.info('Google reports visitBounceRate as %s', bounces) + ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': float(bounces)}, + period_complete_day) + + + def _locale_stats(self, start_date, end_date, period_name, period_complete_day): """ Fetches stats about language and country """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -279,16 +288,72 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Languages", data) + ga_model.update_sitewide_stats(period_name, "Languages", data, period_complete_day) data = {} for result in result_data: data[result[1]] = data.get(result[1], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Country", data) - - - def _social_stats(self, start_date, end_date, period_name): + ga_model.update_sitewide_stats(period_name, "Country", data, period_complete_day) + + + def _download_stats(self, start_date, end_date, period_name, period_complete_day): + """ Fetches stats about language and country """ + import ckan.model as model + + data = {} + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + filters='ga:eventAction==download', + metrics='ga:totalEvents', + sort='-ga:totalEvents', + dimensions="ga:eventLabel", + max_results=10000, + end_date=end_date).execute() + result_data = results.get('rows') + if not result_data: + # We may not have data for this time period, so we need to bail + # early. + log.info("There is no download data for this time period") + return + + def process_result_data(result_data, cached=False): + for result in result_data: + url = result[0].strip() + + # Get package id associated with the resource that has this URL. + q = model.Session.query(model.Resource) + if cached: + r = q.filter(model.Resource.cache_url.like("%s%%" % url)).first() + else: + r = q.filter(model.Resource.url.like("%s%%" % url)).first() + + package_name = r.resource_group.package.name if r else "" + if package_name: + data[package_name] = data.get(package_name, 0) + int(result[1]) + else: + log.warning(u"Could not find resource for URL: {url}".format(url=url)) + continue + + process_result_data(results.get('rows')) + + results = self.service.data().ga().get( + ids='ga:' + self.profile_id, + start_date=start_date, + filters='ga:eventAction==download-cache', + metrics='ga:totalEvents', + sort='-ga:totalEvents', + dimensions="ga:eventLabel", + max_results=10000, + end_date=end_date).execute() + process_result_data(results.get('rows'), cached=False) + + self._filter_out_long_tail(data, MIN_DOWNLOADS) + ga_model.update_sitewide_stats(period_name, "Downloads", data, period_complete_day) + + def _social_stats(self, start_date, end_date, period_name, period_complete_day): """ Finds out which social sites people are referred from """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -304,10 +369,10 @@ if not result[0] == '(not set)': data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, 3) - ga_model.update_sitewide_stats(period_name, "Social sources", data) - - - def _os_stats(self, start_date, end_date, period_name): + ga_model.update_sitewide_stats(period_name, "Social sources", data, period_complete_day) + + + def _os_stats(self, start_date, end_date, period_name, period_complete_day): """ Operating system stats """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -322,17 +387,17 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Operating Systems", data) + ga_model.update_sitewide_stats(period_name, "Operating Systems", data, period_complete_day) data = {} for result in result_data: if int(result[2]) >= MIN_VIEWS: key = "%s %s" % (result[0],result[1]) data[key] = result[2] - ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data) - - - def _browser_stats(self, start_date, end_date, period_name): + ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data, period_complete_day) + + + def _browser_stats(self, start_date, end_date, period_name, period_complete_day): """ Information about browsers and browser versions """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -349,14 +414,14 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Browsers", data) + ga_model.update_sitewide_stats(period_name, "Browsers", data, period_complete_day) data = {} for result in result_data: key = "%s %s" % (result[0], self._filter_browser_version(result[0], result[1])) data[key] = data.get(key, 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Browser versions", data) + ga_model.update_sitewide_stats(period_name, "Browser versions", data, period_complete_day) @classmethod def _filter_browser_version(cls, browser, version_str): @@ -380,7 +445,7 @@ ver = ver[0] + ver[1] + 'X' * num_hidden_digits return ver - def _mobile_stats(self, start_date, end_date, period_name): + def _mobile_stats(self, start_date, end_date, period_name, period_complete_day): """ Info about mobile devices """ results = self.service.data().ga().get( @@ -397,13 +462,13 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Mobile brands", data) + ga_model.update_sitewide_stats(period_name, "Mobile brands", data, period_complete_day) data = {} for result in result_data: data[result[1]] = data.get(result[1], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Mobile devices", data) + ga_model.update_sitewide_stats(period_name, "Mobile devices", data, period_complete_day) @classmethod def _filter_out_long_tail(cls, data, threshold=10): --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -47,6 +47,7 @@ Column('id', types.UnicodeText, primary_key=True, default=make_uuid), Column('period_name', types.UnicodeText), + Column('period_complete_day', types.UnicodeText), Column('stat_name', types.UnicodeText), Column('key', types.UnicodeText), Column('value', types.UnicodeText), ) @@ -134,7 +135,7 @@ return None, publisher_match.groups()[0] return None, None -def update_sitewide_stats(period_name, stat_name, data): +def update_sitewide_stats(period_name, stat_name, data, period_complete_day): for k,v in data.iteritems(): item = model.Session.query(GA_Stat).\ filter(GA_Stat.period_name==period_name).\ @@ -144,11 +145,13 @@ item.period_name = period_name item.key = k item.value = v + item.period_complete_day = period_complete_day model.Session.add(item) else: # create the row values = {'id': make_uuid(), 'period_name': period_name, + 'period_complete_day': period_complete_day, 'key': k, 'value': v, 'stat_name': stat_name @@ -158,10 +161,56 @@ def pre_update_url_stats(period_name): + log.debug("Deleting '%s' records" % period_name) model.Session.query(GA_Url).\ filter(GA_Url.period_name==period_name).delete() - model.Session.query(GA_Url).\ - filter(GA_Url.period_name=='All').delete() + + count = model.Session.query(GA_Url).\ + filter(GA_Url.period_name == 'All').count() + log.debug("Deleting %d 'All' records" % count) + count = model.Session.query(GA_Url).\ + filter(GA_Url.period_name == 'All').delete() + log.debug("Deleted %d 'All' records" % count) + + model.Session.flush() + model.Session.commit() + model.repo.commit_and_remove() + +def post_update_url_stats(): + + """ Check the distinct url field in ga_url and make sure + it has an All record. If not then create one. + + After running this then every URL should have an All + record regardless of whether the URL has an entry for + the month being currently processed. + """ + query = """select url, pageviews::int, visits::int + from ga_url + where url not in (select url from ga_url where period_name ='All')""" + connection = model.Session.connection() + res = connection.execute(query) + + views, visits = {}, {} + # url, views, visits + for row in res: + views[row[0]] = views.get(row[0], 0) + row[1] + visits[row[0]] = visits.get(row[0], 0) + row[2] + + for key in views.keys(): + package, publisher = _get_package_and_publisher(key) + + values = {'id': make_uuid(), + 'period_name': "All", + 'period_complete_day': 0, + 'url': key, + 'pageviews': views[key], + 'visits': visits[key], + 'department_id': publisher, + 'package_id': publisher + } + model.Session.add(GA_Url(**values)) + model.Session.commit() def update_url_stats(period_name, period_complete_day, url_data): @@ -213,8 +262,8 @@ 'period_name': 'All', 'period_complete_day': 0, 'url': url, - 'pageviews': sum([int(e.pageviews) for e in entries]) + old_pageviews, - 'visits': sum([int(e.visits or 0) for e in entries]) + old_visits, + 'pageviews': sum([int(e.pageviews) for e in entries]) + int(old_pageviews), + 'visits': sum([int(e.visits or 0) for e in entries]) + int(old_visits), 'department_id': publisher, 'package_id': package } @@ -340,10 +389,10 @@ ''' for object_type in (GA_Url, GA_Stat, GA_Publisher, GA_ReferralStat): q = model.Session.query(object_type) - if period_name != 'all': + if period_name != 'All': q = q.filter_by(period_name=period_name) q.delete() - model.Session.commit() + model.repo.commit_and_remove() def get_score_for_dataset(dataset_name): ''' --- a/ckanext/ga_report/plugin.py +++ b/ckanext/ga_report/plugin.py @@ -42,6 +42,16 @@ controller='ckanext.ga_report.controller:GaReport', action='csv' ) + map.connect( + '/data/site-usage/downloads', + controller='ckanext.ga_report.controller:GaReport', + action='downloads' + ) + map.connect( + '/data/site-usage/downloads_{month}.csv', + controller='ckanext.ga_report.controller:GaReport', + action='csv_downloads' + ) # GaDatasetReport map.connect( --- a/ckanext/ga_report/templates/ga_report/ga_util.html +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -5,6 +5,14 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip="" > + + + --- a/ckanext/ga_report/templates/ga_report/notes.html +++ b/ckanext/ga_report/templates/ga_report/notes.html @@ -7,7 +7,7 @@

Notes

--- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -27,13 +27,9 @@
- + ${month_selector(c.month, c.months, c.day)} +
@@ -41,14 +37,13 @@
- - - --- a/ckanext/ga_report/templates/ga_report/publisher/read.html +++ b/ckanext/ga_report/templates/ga_report/publisher/read.html @@ -25,12 +25,9 @@
- + + ${month_selector(c.month, c.months, c.day)} +
Publisher Dataset Views
${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name))} + + ${h.link_to(publisher.title, h.url_for(controller='ckanext.ga_report.controller:GaDatasetReport', action='read_publisher', id=publisher.name) + (("?month=" + c.month) if c.month else ''))} ${views}
- + - + - +
Dataset ViewsDownloads
${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))} ${views}${downloads}
--- /dev/null +++ b/ckanext/ga_report/templates/ga_report/site/downloads.html @@ -1,1 +1,59 @@ + + + + Downloads + + +
  • +

    Download

    +

    + Download as CSV
    +

    +
    • + + +     + +
    +

    Downloads

    + ${usage_nav('Downloads')} + + +
    + + ${month_selector(c.month, c.months, c.day)} + + +
    + + + + ${downloads_table(c.downloads)} + + +

    No data

    +

    There is no download data available for this month

    +     +
    + + + + + + + + + + + --- a/ckanext/ga_report/templates/ga_report/site/index.html +++ b/ckanext/ga_report/templates/ga_report/site/index.html @@ -24,13 +24,9 @@
    - + ${month_selector(c.month, c.months, c.day)} +
    --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ loadanalytics = ckanext.ga_report.command:LoadAnalytics initdb = ckanext.ga_report.command:InitDB getauthtoken = ckanext.ga_report.command:GetAuthToken + fixtimeperiods = ckanext.ga_report.command:FixTimePeriods """, )