gitphp 0.2.9.1 :: ckanext-datagovau.git/commitdiff

Google Analytics for CKAN API

3 files changed: (show all)
admin/reset.sh (new)
ckanext/datagovau/controller.py (new)
ckanext/datagovau/plugin.py

file:b/admin/reset.sh (new)

	paster --plugin=ckan db clean --config=development.ini
	echo "drop extension postgis cascade;" \| psql -d ckantest
	paster --plugin=ckan db clean --config=development.ini

	#to initiate for first time instead of load from dump
	#paster --plugin=ckan db init --config=development.in
	#paster --plugin=ckan user add maxious password=snmc email=maxious@gmail.com
	#paster --plugin=ckan sysadmin add maxious
	#paster --plugin=ckan db dump dump.db

	#paster --plugin=ckan db load --config=development.ini dump.db
	paster --plugin=ckan db load --config=development.ini dump.harvest.db
	echo "create extension postgis;" \| psql -d ckantest
	#sleep 2
	paster --plugin=ckan search-index rebuild --config=development.ini
	#rm -r /tmp/pairtree_*

file:b/ckanext/datagovau/controller.py (new)

	import urllib
	import json
	from pprint import pprint
	import logging
	import ckan.logic as logic
	import hashlib
	import threading
	from ckan.common import _, c, request, response
	from pylons import config

	log = logging.getLogger(__name__)

	from ckan.controllers.api import ApiController

	class DGAApiController(ApiController):

	def _post_analytics(self,user,request_obj_type,request_function,request_id):
	if (config.get('googleanalytics.id') != None):
	data = urllib.urlencode({
	"v":1,
	"tid":config.get('googleanalytics.id'),
	"cid":hashlib.md5(user).hexdigest(),
	"t":"event",
	"dh":c.environ['HTTP_HOST'],
	"dp":c.environ['PATH_INFO'],
	"dr":c.environ.get('HTTP_REFERER',''),
	"ec":"CKAN API Request",
	"ea":request_obj_type+request_function,
	"el":request_id,
	})
	log.debug("Sending API Analytics Data: "+data)
	# send analytics asynchronously
	threading.Thread(target=urllib.urlopen,args=("http://www.google-analytics.com/collect", data)).start()


	def action(self, logic_function, ver=None):
	try:
	function = logic.get_action(logic_function)
	except Exception,e:
	log.debug(e)
	pass
	try:
	side_effect_free = getattr(function, 'side_effect_free', False)
	request_data = self._get_request_data(try_url_params=side_effect_free)
	if isinstance(request_data, dict):
	id = request_data.get('id','')
	if 'q' in request_data.keys():
	id = request_data['q']
	if 'query' in request_data.keys():
	id = request_data['query']
	self._post_analytics(c.user,logic_function,'', id)
	except Exception,e:
	print log.debug(e)
	pass

	return ApiController.action(self,logic_function, ver)

	def list(self, ver=None, register=None, subregister=None, id=None):
	self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"list",id)
	return ApiController.list(self,ver, register, subregister, id)
	def show(self, ver=None, register=None, subregister=None, id=None, id2=None):
	self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"show",id)
	return ApiController.show(self,ver, register, subregister, id,id2)
	def update(self, ver=None, register=None, subregister=None, id=None, id2=None):
	self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"update",id)
	return ApiController.update(self,ver, register, subregister, id,id2)
	def delete(self, ver=None, register=None, subregister=None, id=None, id2=None):
	self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"delete",id)
	return ApiController.delete(self,ver, register, subregister, id,id2)
	def search(self, ver=None, register=None):
	id = None
	try:
	params = MultiDict(self._get_search_params(request.params))
	if 'q' in params.keys():
	id = params['q']
	if 'query' in params.keys():
	id = params['query']
	except ValueError, e:
	print str(e)
	pass
	self._post_analytics(c.user,register,"search",id)

file:a/ckanext/datagovau/plugin.py -> file:b/ckanext/datagovau/plugin.py

import logging	import logging

import ckan.plugins as plugins	import ckan.plugins as plugins
import ckan.lib as lib	import ckan.lib as lib
import ckan.lib.dictization.model_dictize as model_dictize	import ckan.lib.dictization.model_dictize as model_dictize
import ckan.plugins.toolkit as tk	import ckan.plugins.toolkit as tk
import ckan.model as model	import ckan.model as model
from pylons import config	from pylons import config
	from routes.mapper import SubMapper, Mapper as _Mapper

from sqlalchemy import orm	from sqlalchemy import orm
import ckan.model	import ckan.model

#parse the activity feed for last active non-system user	#parse the activity feed for last active non-system user
def get_last_active_user(id):	def get_last_active_user(id):
system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')})	system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')})
user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']]	user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']]
user = None	user = None
if len(user_list) > 0:	if len(user_list) > 0:
user = user_list[0].get('user_id', None)	user = user_list[0].get('user_id', None)
if user is None:	if user is None:
return system_user	return system_user
else:	else:
return lib.helpers.get_action('user_show',{'id':user})	return lib.helpers.get_action('user_show',{'id':user})

# get user created datasets and those they have edited	# get user created datasets and those they have edited
def get_user_datasets(user_dict):	def get_user_datasets(user_dict):
created_datasets_list = user_dict['datasets']	created_datasets_list = user_dict['datasets']
active_datasets_list = [x['data']['package'] for x in	active_datasets_list = [x['data']['package'] for x in
lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')]	lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')]
return created_datasets_list + active_datasets_list	return created_datasets_list + active_datasets_list

def get_dga_stats():
connection = model.Session.connection()
res = connection.execute("SELECT 'organization', count() from \"group\" where type = 'organization' and state = 'active' union select 'package', count() from package where state='active' or state='draft' or state='draft-complete' union select 'resource', count(*) from resource where state='active' union select name\|\|role, 0 from user_object_role inner join \"user\" on user_object_role.user_id = \"user\".id where name not in ('logged_in','visitor') group by name,role;")
return res


def get_activity_counts():
connection = model.Session.connection()
res = connection.execute("select to_char(timestamp, 'YYYY-MM') as month,activity_type, count(*) from activity group by month, activity_type order by month;").fetchall();
return res


class DataGovAuPlugin(plugins.SingletonPlugin,	class DataGovAuPlugin(plugins.SingletonPlugin,
tk.DefaultDatasetForm):	tk.DefaultDatasetForm):
'''An example IDatasetForm CKAN plugin.	'''An example IDatasetForm CKAN plugin.

Uses a tag vocabulary to add a custom metadata field to datasets.	Uses a tag vocabulary to add a custom metadata field to datasets.

'''	'''
plugins.implements(plugins.IConfigurer, inherit=False)	plugins.implements(plugins.IConfigurer, inherit=False)
plugins.implements(plugins.IDatasetForm, inherit=False)	plugins.implements(plugins.IDatasetForm, inherit=False)
plugins.implements(plugins.ITemplateHelpers, inherit=False)	plugins.implements(plugins.ITemplateHelpers, inherit=False)
	plugins.implements(plugins.IRoutes, inherit=True)

	def before_map(self, map):

	# Helpers to reduce code clutter
	GET = dict(method=['GET'])
	PUT = dict(method=['PUT'])
	POST = dict(method=['POST'])
	DELETE = dict(method=['DELETE'])
	GET_POST = dict(method=['GET', 'POST'])
	# intercept API calls that we want to capture analytics on
	register_list = [
	'package',
	'dataset',
	'resource',
	'tag',
	'group',
	'related',
	'revision',
	'licenses',
	'rating',
	'user',
	'activity'
	]
	register_list_str = '\|'.join(register_list)
	# /api ver 3 or none
	with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/3\|}',
	ver='/3') as m:
	m.connect('/action/{logic_function}', action='action',
	conditions=GET_POST)

	# /api ver 1, 2, 3 or none
	with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1\|/2\|/3\|}',
	ver='/1') as m:
	m.connect('/search/{register}', action='search')

	# /api/rest ver 1, 2 or none
	with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1\|/2\|}',
	ver='/1', requirements=dict(register=register_list_str)
	) as m:

	m.connect('/rest/{register}', action='list', conditions=GET)
	m.connect('/rest/{register}', action='create', conditions=POST)
	m.connect('/rest/{register}/{id}', action='show', conditions=GET)
	m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
	m.connect('/rest/{register}/{id}', action='update', conditions=POST)
	m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE)

	return map

def update_config(self, config):	def update_config(self, config):
# Add this plugin's templates dir to CKAN's extra_template_paths, so	# Add this plugin's templates dir to CKAN's extra_template_paths, so
# that CKAN will use this plugin's custom templates.	# that CKAN will use this plugin's custom templates.
# here = os.path.dirname(__file__)	# here = os.path.dirname(__file__)
# rootdir = os.path.dirname(os.path.dirname(here))	# rootdir = os.path.dirname(os.path.dirname(here))

tk.add_template_directory(config, 'templates')	tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'theme/public')	tk.add_public_directory(config, 'theme/public')
tk.add_resource('theme/public', 'ckanext-datagovau')	tk.add_resource('theme/public', 'ckanext-datagovau')
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json'	# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json'

def get_helpers(self):	def get_helpers(self):
return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets, 'get_dga_stats': get_dga_stats, 'get_activity_counts': get_activity_counts}	return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets}

def is_fallback(self):	def is_fallback(self):
# Return True to register this plugin as the default handler for	# Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin.	# package types not handled by any other IDatasetForm plugin.
return True	return True

def package_types(self):	def package_types(self):
# This plugin doesn't handle any special package types, it just	# This plugin doesn't handle any special package types, it just
# registers itself as the default (above).	# registers itself as the default (above).
return []	return []


def create_package_schema(self):	def create_package_schema(self):
schema = super(DataGovAuPlugin, self).create_package_schema()	schema = super(DataGovAuPlugin, self).create_package_schema()
schema = self._modify_package_schema(schema)	schema = self._modify_package_schema(schema)
return schema	return schema

def update_package_schema(self):	def update_package_schema(self):
schema = super(DataGovAuPlugin, self).update_package_schema()	schema = super(DataGovAuPlugin, self).update_package_schema()
schema = self._modify_package_schema(schema)	schema = self._modify_package_schema(schema)
return schema	return schema

def show_package_schema(self):	def show_package_schema(self):
schema = super(DataGovAuPlugin, self).show_package_schema()	schema = super(DataGovAuPlugin, self).show_package_schema()

# Don't show vocab tags mixed in with normal 'free' tags	# Don't show vocab tags mixed in with normal 'free' tags
# (e.g. on dataset pages, or on the search page)	# (e.g. on dataset pages, or on the search page)
schema['tags']['__extras'].append(tk.get_converter('free_tags_only'))	schema['tags']['__extras'].append(tk.get_converter('free_tags_only'))

# Add our custom_text field to the dataset schema.	# Add our custom_text field to the dataset schema.
# ignore_missing == optional	# ignore_missing == optional
# ignore_empty == mandatory but not for viewing	# ignore_empty == mandatory but not for viewing
# !!! always convert_from_extras first	# !!! always convert_from_extras first
schema.update({	schema.update({
'agency_program': [tk.get_converter('convert_from_extras'),	'agency_program': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_missing')],	tk.get_validator('ignore_missing')],
'contact_point': [tk.get_converter('convert_from_extras'),	'contact_point': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'spatial_coverage': [tk.get_converter('convert_from_extras'),	'spatial_coverage': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'granularity': [tk.get_converter('convert_from_extras'),	'granularity': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'jurisdiction': [tk.get_converter('convert_from_extras'),	'jurisdiction': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'temporal_coverage': [tk.get_converter('convert_from_extras'),	'temporal_coverage': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'data_state': [tk.get_converter('convert_from_extras'),	'data_state': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')],	tk.get_validator('ignore_empty')],
'update_freq': [tk.get_converter('convert_from_extras'),	'update_freq': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')]	tk.get_validator('ignore_empty')]
})	})
return schema	return schema

def _modify_package_schema(self, schema):	def _modify_package_schema(self, schema):
# Add our custom_test metadata field to the schema, this one will use	# Add our custom_test metadata field to the schema, this one will use
# convert_to_extras instead of convert_to_tags.	# convert_to_extras instead of convert_to_tags.
# ignore_missing == optional	# ignore_missing == optional
# not_empty == mandatory, enforced here while modifying	# not_empty == mandatory, enforced here while modifying

schema.update({	schema.update({
'agency_program': [tk.get_validator('ignore_missing'),	'agency_program': [tk.get_validator('ignore_missing'),
tk.get_converter('convert_to_extras')],	tk.get_converter('convert_to_extras')],
'contact_point': [tk.get_converter('convert_to_extras'),	'contact_point': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'spatial_coverage': [tk.get_converter('convert_to_extras'),	'spatial_coverage': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'granularity': [tk.get_converter('convert_to_extras'),	'granularity': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'jurisdiction': [tk.get_converter('convert_to_extras'),	'jurisdiction': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'temporal_coverage': [tk.get_converter('convert_to_extras'),	'temporal_coverage': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'data_state': [tk.get_converter('convert_to_extras'),	'data_state': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')],	tk.get_validator('not_empty')],
'update_freq': [tk.get_converter('convert_to_extras'),	'update_freq': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')]	tk.get_validator('not_empty')]
})	})
return schema	return schema

# These methods just record how many times they're called, for testing	# These methods just record how many times they're called, for testing
# purposes.	# purposes.
# TODO: It might be better to test that custom templates returned by	# TODO: It might be better to test that custom templates returned by
# these methods are actually used, not just that the methods get	# these methods are actually used, not just that the methods get
# called.	# called.

def setup_template_variables(self, context, data_dict):	def setup_template_variables(self, context, data_dict):
return super(DataGovAuPlugin, self).setup_template_variables(	return super(DataGovAuPlugin, self).setup_template_variables(
context, data_dict)	context, data_dict)

def new_template(self):	def new_template(self):
return super(DataGovAuPlugin, self).new_template()	return super(DataGovAuPlugin, self).new_template()

def read_template(self):	def read_template(self):
return super(DataGovAuPlugin, self).read_template()	return super(DataGovAuPlugin, self).read_template()

def edit_template(self):	def edit_template(self):
return super(DataGovAuPlugin, self).edit_template()	return super(DataGovAuPlugin, self).edit_template()

def search_template(self):	def search_template(self):
return super(DataGovAuPlugin, self).search_template()	return super(DataGovAuPlugin, self).search_template()

def history_template(self):	def history_template(self):
return super(DataGovAuPlugin, self).history_template()	return super(DataGovAuPlugin, self).history_template()

def package_form(self):	def package_form(self):
return super(DataGovAuPlugin, self).package_form()	return super(DataGovAuPlugin, self).package_form()