From: David Read Date: Wed, 21 Nov 2012 14:29:53 +0000 Subject: Avoid problem when testing. X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-ga-report.git&a=commitdiff&h=4dc59ec64eb44f79fb66fe8347e2456cb1b5cae0 --- Avoid problem when testing. --- --- a/ckanext/ga_report/controller.py +++ b/ckanext/ga_report/controller.py @@ -22,29 +22,12 @@ def _month_details(cls): - ''' - Returns a list of all the periods for which we have data, unfortunately - knows too much about the type of the cls being passed as GA_Url has a - more complex query - - This may need extending if we add a period_name to the stats - ''' + '''Returns a list of all the month names''' months = [] - day = None - - vals = model.Session.query(cls.period_name,cls.period_complete_day)\ - .filter(cls.period_name!='All').distinct(cls.period_name)\ - .order_by("period_name desc").all() - if vals and vals[0][1]: - day = int(vals[0][1]) - ordinal = 'th' if 11 <= day <= 13 \ - else {1:'st',2:'nd',3:'rd'}.get(day % 10, 'th') - day = "{day}{ordinal}".format(day=day, ordinal=ordinal) - + vals = model.Session.query(cls.period_name).filter(cls.period_name!='All').distinct().all() for m in vals: months.append( (m[0], _get_month_name(m[0]))) - - return months, day + return sorted(months, key=operator.itemgetter(0), reverse=True) class GaReport(BaseController): @@ -73,7 +56,7 @@ # Get the month details by fetching distinct values and determining the # month names from the values. - c.months, c.day = _month_details(GA_Stat) + c.months = _month_details(GA_Stat) # Work out which month to show, based on query params of the first item c.month_desc = 'all months' @@ -237,7 +220,7 @@ # Get the month details by fetching distinct values and determining the # month names from the values. - c.months, c.day = _month_details(GA_Url) + c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', '') @@ -295,7 +278,7 @@ # Get the month details by fetching distinct values and determining the # month names from the values. - c.months, c.day = _month_details(GA_Url) + c.months = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', '') --- a/ckanext/ga_report/download_analytics.py +++ b/ckanext/ga_report/download_analytics.py @@ -126,7 +126,7 @@ ga_model.update_publisher_stats(period_name) # about 30 seconds. log.info('Downloading and storing analytics for site-wide stats') - self.sitewide_stats( period_name, period_complete_day ) + self.sitewide_stats( period_name ) log.info('Downloading and storing analytics for social networks') self.update_social_info(period_name, start_date, end_date) @@ -194,7 +194,7 @@ if 'url' in data: ga_model.update_url_stats(period_name, period_complete_day, data['url']) - def sitewide_stats(self, period_name, period_complete_day): + def sitewide_stats(self, period_name): import calendar year, month = period_name.split('-') _, last_day_of_month = calendar.monthrange(int(year), int(month)) @@ -205,7 +205,7 @@ '_locale_stats', '_browser_stats', '_mobile_stats'] for f in funcs: log.info('Downloading analytics for %s' % f.split('_')[1]) - getattr(self, f)(start_date, end_date, period_name, period_complete_day) + getattr(self, f)(start_date, end_date, period_name) def _get_results(result_data, f): data = {} @@ -214,7 +214,7 @@ data[key] = data.get(key,0) + result[1] return data - def _totals_stats(self, start_date, end_date, period_name, period_complete_day): + def _totals_stats(self, start_date, end_date, period_name): """ Fetches distinct totals, total pageviews etc """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -224,8 +224,7 @@ max_results=10000, end_date=end_date).execute() result_data = results.get('rows') - ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}, - period_complete_day) + ga_model.update_sitewide_stats(period_name, "Totals", {'Total page views': result_data[0][0]}) results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -240,7 +239,7 @@ 'New visits': result_data[0][2], 'Total visits': result_data[0][3], } - ga_model.update_sitewide_stats(period_name, "Totals", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Totals", data) # Bounces from / or another configurable page. path = '/%s%s' % (config.get('googleanalytics.account'), @@ -262,11 +261,10 @@ bounces, total = [float(x) for x in result_data[0][1:]] pct = 100 * bounces/total log.info('%d bounces from %d total == %s', bounces, total, pct) - ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': pct}, - period_complete_day) - - - def _locale_stats(self, start_date, end_date, period_name, period_complete_day): + ga_model.update_sitewide_stats(period_name, "Totals", {'Bounce rate (home page)': pct}) + + + def _locale_stats(self, start_date, end_date, period_name): """ Fetches stats about language and country """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -281,16 +279,16 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Languages", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Languages", data) data = {} for result in result_data: data[result[1]] = data.get(result[1], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Country", data, period_complete_day) - - - def _social_stats(self, start_date, end_date, period_name, period_complete_day): + ga_model.update_sitewide_stats(period_name, "Country", data) + + + def _social_stats(self, start_date, end_date, period_name): """ Finds out which social sites people are referred from """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -306,10 +304,10 @@ if not result[0] == '(not set)': data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, 3) - ga_model.update_sitewide_stats(period_name, "Social sources", data, period_complete_day) - - - def _os_stats(self, start_date, end_date, period_name, period_complete_day): + ga_model.update_sitewide_stats(period_name, "Social sources", data) + + + def _os_stats(self, start_date, end_date, period_name): """ Operating system stats """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -324,17 +322,17 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Operating Systems", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Operating Systems", data) data = {} for result in result_data: if int(result[2]) >= MIN_VIEWS: key = "%s %s" % (result[0],result[1]) data[key] = result[2] - ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data, period_complete_day) - - - def _browser_stats(self, start_date, end_date, period_name, period_complete_day): + ga_model.update_sitewide_stats(period_name, "Operating Systems versions", data) + + + def _browser_stats(self, start_date, end_date, period_name): """ Information about browsers and browser versions """ results = self.service.data().ga().get( ids='ga:' + self.profile_id, @@ -351,14 +349,14 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Browsers", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Browsers", data) data = {} for result in result_data: key = "%s %s" % (result[0], self._filter_browser_version(result[0], result[1])) data[key] = data.get(key, 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Browser versions", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Browser versions", data) @classmethod def _filter_browser_version(cls, browser, version_str): @@ -382,7 +380,7 @@ ver = ver[0] + ver[1] + 'X' * num_hidden_digits return ver - def _mobile_stats(self, start_date, end_date, period_name, period_complete_day): + def _mobile_stats(self, start_date, end_date, period_name): """ Info about mobile devices """ results = self.service.data().ga().get( @@ -399,13 +397,13 @@ for result in result_data: data[result[0]] = data.get(result[0], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Mobile brands", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Mobile brands", data) data = {} for result in result_data: data[result[1]] = data.get(result[1], 0) + int(result[2]) self._filter_out_long_tail(data, MIN_VIEWS) - ga_model.update_sitewide_stats(period_name, "Mobile devices", data, period_complete_day) + ga_model.update_sitewide_stats(period_name, "Mobile devices", data) @classmethod def _filter_out_long_tail(cls, data, threshold=10): --- a/ckanext/ga_report/ga_model.py +++ b/ckanext/ga_report/ga_model.py @@ -47,7 +47,6 @@ Column('id', types.UnicodeText, primary_key=True, default=make_uuid), Column('period_name', types.UnicodeText), - Column('period_complete_day', types.UnicodeText), Column('stat_name', types.UnicodeText), Column('key', types.UnicodeText), Column('value', types.UnicodeText), ) @@ -135,7 +134,7 @@ return None, publisher_match.groups()[0] return None, None -def update_sitewide_stats(period_name, stat_name, data, period_complete_day): +def update_sitewide_stats(period_name, stat_name, data): for k,v in data.iteritems(): item = model.Session.query(GA_Stat).\ filter(GA_Stat.period_name==period_name).\ @@ -145,13 +144,11 @@ item.period_name = period_name item.key = k item.value = v - item.period_complete_day = period_complete_day model.Session.add(item) else: # create the row values = {'id': make_uuid(), 'period_name': period_name, - 'period_complete_day': period_complete_day, 'key': k, 'value': v, 'stat_name': stat_name --- a/ckanext/ga_report/helpers.py +++ b/ckanext/ga_report/helpers.py @@ -50,9 +50,12 @@ dataset = model.Package.get(ga_url.url[len('/dataset/'):]) if dataset and not dataset.state == 'active': dataset = None - count += 1 - if count > 10: - break + # When testing, it is possible that top datasets are not available + # so only go round this loop a few times before falling back on + # a random dataset. + count += 1 + if count > 10: + break if not dataset: # fallback dataset = model.Session.query(model.Package)\ --- a/ckanext/ga_report/templates/ga_report/ga_util.html +++ b/ckanext/ga_report/templates/ga_report/ga_util.html @@ -5,14 +5,6 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip="" > - - - --- a/ckanext/ga_report/templates/ga_report/publisher/index.html +++ b/ckanext/ga_report/templates/ga_report/publisher/index.html @@ -27,9 +27,13 @@
+
--- a/ckanext/ga_report/templates/ga_report/publisher/read.html +++ b/ckanext/ga_report/templates/ga_report/publisher/read.html @@ -25,9 +25,12 @@
- - ${month_selector(c.month, c.months, c.day)} - + + - ${month_selector(c.month, c.months, c.day)} - + + + +