From: Alex Sadleir Date: Mon, 06 Jan 2014 12:34:39 +0000 Subject: Move API google analytics and AGLS metadata to seperate extensions X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=4a98f04e196202e68af2bfe88e33018a1d6dff2d --- Move API google analytics and AGLS metadata to seperate extensions --- --- a/README.rst +++ b/README.rst @@ -2,15 +2,12 @@ It comprises: -* A CKAN Extension "plugin" at ``ckanext/datagovau/plugin.py`` which, when - loaded, overrides various settings in the core ``ini``-file to provide: - * A path to local customisations of the core templates to include AGLS/Dublin Core minimum metadata - * A custom Package edit form that defaults to cc-by licence - * A custom n3/rdf output format - * Replaces links with http/https protocol independent versions - * Provides HTML to users to embed data previews on their own website +* A custom Package edit form that defaults to cc-by licence +* Replaces links with http/https protocol independent versions +* Provides HTML to users to embed data previews on their own website +* A cut down licenses.json file -* A cut down licenses.json file +This extension is complimented by ckanext-agls for AGLS metadata and ckanext-googleanalytics for Google Analytics tracking of API usage Installation ============ --- a/ckanext/datagovau/controller.py +++ /dev/null @@ -1,84 +1,1 @@ -import urllib -import json -from pprint import pprint -import logging -import ckan.logic as logic -import hashlib -import threading -from ckan.common import _, c, request, response -from pylons import config -from webob.multidict import UnicodeMultiDict -from paste.util.multidict import MultiDict -log = logging.getLogger(__name__) - -from ckan.controllers.api import ApiController - -class DGAApiController(ApiController): - - def _post_analytics(self,user,request_obj_type,request_function,request_id): - if (config.get('googleanalytics.id') != None): - data = urllib.urlencode({ - "v":1, - "tid":config.get('googleanalytics.id'), - "cid":hashlib.md5(user).hexdigest(), - "t":"event", - "dh":c.environ['HTTP_HOST'], - "dp":c.environ['PATH_INFO'], - "dr":c.environ.get('HTTP_REFERER',''), - "ec":"CKAN API Request", - "ea":request_obj_type+request_function, - "el":request_id, - }) - log.debug("Sending API Analytics Data: "+data) - # send analytics asynchronously - threading.Thread(target=urllib.urlopen,args=("http://www.google-analytics.com/collect", data)).start() - - - def action(self, logic_function, ver=None): - try: - function = logic.get_action(logic_function) - except Exception,e: - log.debug(e) - pass - try: - side_effect_free = getattr(function, 'side_effect_free', False) - request_data = self._get_request_data(try_url_params=side_effect_free) - if isinstance(request_data, dict): - id = request_data.get('id','') - if 'q' in request_data.keys(): - id = request_data['q'] - if 'query' in request_data.keys(): - id = request_data['query'] - self._post_analytics(c.user,logic_function,'', id) - except Exception,e: - print log.debug(e) - pass - - return ApiController.action(self,logic_function, ver) - - def list(self, ver=None, register=None, subregister=None, id=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"list",id) - return ApiController.list(self,ver, register, subregister, id) - def show(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"show",id) - return ApiController.show(self,ver, register, subregister, id,id2) - def update(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"update",id) - return ApiController.update(self,ver, register, subregister, id,id2) - def delete(self, ver=None, register=None, subregister=None, id=None, id2=None): - self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"delete",id) - return ApiController.delete(self,ver, register, subregister, id,id2) - def search(self, ver=None, register=None): - id = None - try: - params = MultiDict(self._get_search_params(request.params)) - if 'q' in params.keys(): - id = params['q'] - if 'query' in params.keys(): - id = params['query'] - except ValueError, e: - print str(e) - pass - self._post_analytics(c.user,register,"search",id) - --- a/ckanext/datagovau/plugin.py +++ b/ckanext/datagovau/plugin.py @@ -6,22 +6,9 @@ import ckan.plugins.toolkit as tk import ckan.model as model from pylons import config -from routes.mapper import SubMapper, Mapper as _Mapper from sqlalchemy import orm import ckan.model - -#parse the activity feed for last active non-system user -def get_last_active_user(id): - system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')}) - user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']] - user = None - if len(user_list) > 0: - user = user_list[0].get('user_id', None) - if user is None: - return system_user - else: - return lib.helpers.get_action('user_show',{'id':user}) # get user created datasets and those they have edited def get_user_datasets(user_dict): @@ -43,57 +30,7 @@ ''' plugins.implements(plugins.IConfigurer, inherit=False) - plugins.implements(plugins.IDatasetForm, inherit=False) plugins.implements(plugins.ITemplateHelpers, inherit=False) - plugins.implements(plugins.IRoutes, inherit=True) - - def before_map(self, map): - - # Helpers to reduce code clutter - GET = dict(method=['GET']) - PUT = dict(method=['PUT']) - POST = dict(method=['POST']) - DELETE = dict(method=['DELETE']) - GET_POST = dict(method=['GET', 'POST']) - # intercept API calls that we want to capture analytics on - register_list = [ - 'package', - 'dataset', - 'resource', - 'tag', - 'group', - 'related', - 'revision', - 'licenses', - 'rating', - 'user', - 'activity' - ] - register_list_str = '|'.join(register_list) - # /api ver 3 or none - with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/3|}', - ver='/3') as m: - m.connect('/action/{logic_function}', action='action', - conditions=GET_POST) - - # /api ver 1, 2, 3 or none - with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1|/2|/3|}', - ver='/1') as m: - m.connect('/search/{register}', action='search') - - # /api/rest ver 1, 2 or none - with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1|/2|}', - ver='/1', requirements=dict(register=register_list_str) - ) as m: - - m.connect('/rest/{register}', action='list', conditions=GET) - m.connect('/rest/{register}', action='create', conditions=POST) - m.connect('/rest/{register}/{id}', action='show', conditions=GET) - m.connect('/rest/{register}/{id}', action='update', conditions=PUT) - m.connect('/rest/{register}/{id}', action='update', conditions=POST) - m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE) - - return map def update_config(self, config): # Add this plugin's templates dir to CKAN's extra_template_paths, so @@ -107,112 +44,6 @@ # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' def get_helpers(self): - return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets} - - def is_fallback(self): - # Return True to register this plugin as the default handler for - # package types not handled by any other IDatasetForm plugin. - return True - - def package_types(self): - # This plugin doesn't handle any special package types, it just - # registers itself as the default (above). - return [] + return {'get_user_datasets': get_user_datasets} - def create_package_schema(self): - schema = super(DataGovAuPlugin, self).create_package_schema() - schema = self._modify_package_schema(schema) - return schema - - def update_package_schema(self): - schema = super(DataGovAuPlugin, self).update_package_schema() - schema = self._modify_package_schema(schema) - return schema - - def show_package_schema(self): - schema = super(DataGovAuPlugin, self).show_package_schema() - - # Don't show vocab tags mixed in with normal 'free' tags - # (e.g. on dataset pages, or on the search page) - schema['tags']['__extras'].append(tk.get_converter('free_tags_only')) - - # Add our custom_text field to the dataset schema. - # ignore_missing == optional - # ignore_empty == mandatory but not for viewing - # !!! always convert_from_extras first - schema.update({ - 'agency_program': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_missing')], - 'contact_point': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'spatial_coverage': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'granularity': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'jurisdiction': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'temporal_coverage': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'data_state': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')], - 'update_freq': [tk.get_converter('convert_from_extras'), - tk.get_validator('ignore_empty')] - }) - return schema - - def _modify_package_schema(self, schema): - # Add our custom_test metadata field to the schema, this one will use - # convert_to_extras instead of convert_to_tags. - # ignore_missing == optional - # not_empty == mandatory, enforced here while modifying - - schema.update({ - 'agency_program': [tk.get_validator('ignore_missing'), - tk.get_converter('convert_to_extras')], - 'contact_point': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'spatial_coverage': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'granularity': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'jurisdiction': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'temporal_coverage': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'data_state': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')], - 'update_freq': [tk.get_converter('convert_to_extras'), - tk.get_validator('not_empty')] - }) - return schema - - # These methods just record how many times they're called, for testing - # purposes. - # TODO: It might be better to test that custom templates returned by - # these methods are actually used, not just that the methods get - # called. - - def setup_template_variables(self, context, data_dict): - return super(DataGovAuPlugin, self).setup_template_variables( - context, data_dict) - - def new_template(self): - return super(DataGovAuPlugin, self).new_template() - - def read_template(self): - return super(DataGovAuPlugin, self).read_template() - - def edit_template(self): - return super(DataGovAuPlugin, self).edit_template() - - def search_template(self): - return super(DataGovAuPlugin, self).search_template() - - def history_template(self): - return super(DataGovAuPlugin, self).history_template() - - def package_form(self): - return super(DataGovAuPlugin, self).package_form() - - --- a/ckanext/datagovau/templates/package/read.html +++ b/ckanext/datagovau/templates/package/read.html @@ -4,127 +4,6 @@
{{ h.disqus_comments() }}
-{% endblock %} -{% block package_additional_info %} -
-

{{ _('Additional Info') }}

- - - - - - - - - - - {# Add our custom field to the dataset read page. #} - {% if pkg.get('metadata_created') %} - - - - - {% endif %} - {% if pkg.get('metadata_updated') %} - - - - - {% endif %} - {% if pkg.get('agency_program') %} - - - - - {% endif %} - {% if pkg.get('url') %} - - - - - {% endif %} - - {% if pkg.get('contact_point') %} - - - - - {% endif %} - {% if pkg.get('spatial_coverage') %} - - - - - - {% endif %} - {% if pkg.get('granularity') %} - - - - - - {% endif %} - {% if pkg.get('jurisdiction') %} - - - - - {% endif %} - {% if pkg.get('temporal_coverage') %} - - - - - {% endif %} - {% if pkg.get('data_state') %} - - - - - {% endif %} - {% if pkg.get('update_freq') %} - - - - - - {% endif %} - -{% set email =h.get_last_active_user(c.pkg_dict['id']).get('email','') %} - - - - - - - - - - - - - - - - - - -
{{ _('Field') }}{{ _('Value') }}
Date Published{{ pkg.metadata_created.split("T")[0] }}
Date Updated{{ pkg.metadata_updated.split("T")[0] }}
Agency Program {{ pkg.agency_program }}
{{ _('Source') }}{{ h.link_to(pkg.get('url'), - pkg.get('url'), rel='dct:source', target='_blank') }} -
Contact Point {{ h.mail_to(email_address=pkg.contact_point, - name=pkg.contact_point) }}
Geospatial Coverage {{ pkg.spatial_coverage }}
Data Granularity {{ pkg.granularity }}
Government Jurisdiction {{ pkg.jurisdiction }}
Temporal Coverage {{ pkg.temporal_coverage }}
Data State {{ pkg.data_state }}
Update Frequency {{ pkg.update_freq }}
{{ _('Maintainer') }}{{ - h.mail_to(email_address=(email or ' '), name=h.get_last_active_user(c.pkg_dict['id']).get("display_name",'')) }} -
Publisher/Agency{{ c.pkg_dict['organization']['title']}} -
TypeDataset
LanguageEnglish
-
- - {% endblock %} {% block secondary_content %} --- a/ckanext/datagovau/templates/package/read.n3 +++ /dev/null @@ -1,45 +1,1 @@ -@prefix : . -@prefix dcat: . -@prefix dct: . -@prefix foaf: . -@prefix owl: . -@prefix rdf: . -<${ h.url_for(controller='package',action='read',id=c.pkg_dict['name'], qualified=True)}> -a dcat:Dataset; - dct:description "${c.pkg_dict['notes']}"; - dct:identifier "${c.pkg_dict['name']}"; - dct:relation [ - rdf:value ""; - :label "change_note" ], - [ - rdf:value ""; - :label "definition_note" ], - [ - rdf:value ""; - :label "editorial_note" ], - [ - rdf:value ""; - :label "example_note" ], - [ - rdf:value ""; - :label "history_note" ], - [ - rdf:value ""; - :label "scope_note" ], - [ - rdf:value ""; - :label "skos_note" ], - [ - rdf:value ""; - :label "temporal_granularity" ], - [ - rdf:value ""; - :label "type_of_dataset" ], - [ - rdf:value ""; - :label "update_frequency" ]; - dct:title "${c.pkg_dict['title']}"; - :label "${c.pkg_dict['name']}"; - = ; - foaf:homepage . --- a/ckanext/datagovau/templates/package/read.rdf +++ /dev/null @@ -1,133 +1,1 @@ - - - - ${c.pkg_dict['name']} - ${c.pkg_dict['title']} - - - Dataset - ${c.pkg_dict['notes']} - ${c.pkg_dict['metadata_created']} - ${c.pkg_dict['metadata_modified']} - - - - - - - ${c.pkg_dict['license_id']} - "${c.pkg_dict['license_id']}" - - - - ${ tag_dict["name"] } - - - - - - - ${rsc_dict.get('name')} - - ${rsc_dict.get('description')} - - ${rsc_dict.get('created')} - ${rsc_dict.get('revision_timestamp')} - ${rsc_dict.get('size')} - ${rsc_dict.get('mimetype')} - ${rsc_dict.get('size')} bytes - - - ${rsc_dict.get('format')} - ${rsc_dict.get('format')} - - - ${rsc_dict.get('name')} - - - - - - - - - ${ c.pkg_dict['organization']['title'] } - - - - - ${ c.pkg_dict['organization']['title'] } - - - - - ${h.get_last_active_user(c.pkg_dict['id'])["display_name"]} - - - - - - - - ${extra_dict.get('key','')} - ${extra_dict.get('value','')} - - - - en - - ${c.pkg_dict.contact_point } - - - ${ c.pkg_dict.spatial } - - - ${ c.pkg_dict.spatial_coverage } - - - ${ c.pkg_dict.jurisdiction } - ${ c.pkg_dict.get('temporal_coverage') } - - - Data State - ${ c.pkg_dict.get('data_state') } - - - - - Update Frequency - ${ c.pkg_dict.get('update_freq') } - - - - - Agency Program - ${ c.pkg_dict.get('agency_program') } - - - - - Data Granularity - ${ c.pkg_dict.get('granularity') } - - - - - --- a/ckanext/datagovau/templates/package/read_base.html +++ /dev/null @@ -1,47 +1,1 @@ -{% ckan_extends %} -{% block links -%} -{{ super() }} - - - -{% endblock -%} - -{% block head_extras -%} -{{ super() }} - - - - - - - - - - - - - - - - -{% endblock -%} - -{% block package_info %} -
-
-

{{ pkg.title or pkg.name }}

-
-
-
{{ _('Followers') }}
-
{{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}
-
-
- -
-
-{% endblock %} - - --- a/ckanext/datagovau/templates/package/resource_read.html +++ /dev/null @@ -1,35 +1,1 @@ -{% ckan_extends %} -{% block links -%} -{{ super() }} - - - -{% endblock -%} - -{% block head_extras -%} -{{ super() }} - - - - - - - - - - - - - - - - - - - - -{% endblock -%} - - - --- a/ckanext/datagovau/templates/package/snippets/package_basic_fields.html +++ b/ckanext/datagovau/templates/package/snippets/package_basic_fields.html @@ -1,42 +1,4 @@ {% ckan_extends %} - -{% block package_basic_fields_org %} - -{{ super() }} - - -{{ form.input('agency_program', label=_('Agency Program'), id='field-agency_program', -placeholder=_('Name of the Agency Program that generated the data if relevant'), -value=data.agency_program, error=errors.agency_program, classes=['control-medium']) }} - -{{ form.input('contact_point', label=_('Contact Point'), id='field-contact_point', -placeholder=_('Email address for questions about content of dataset'), -value=data.contact_point, error=errors.contact_point, classes=['control-medium']) }} - -{{ form.input('spatial_coverage', label=_('Geospatial Coverage'), id='field-spatial_coverage', -placeholder=_('The geographical area covered by the data. Eg: Whole of Australia, New South Wales, Canberra'), -value=data.spatial_coverage, error=errors.spatial_coverage, classes=['control-medium']) }} - -{{ form.input('granularity', label=_('Data Granularity'), id='field-granularity', -placeholder=_('Granularity in time, place etc. Eg: incidents reports on daily basis by location.'), -value=data.granularity, error=errors.granularity, classes=['control-medium']) }} - -{{ form.input('jurisdiction', label=_('Government Jurisdiction'), id='field-jurisdiction', -placeholder=_('New South Wales, Glenorchy City Council, Federal'), -value=data.jurisdiction, error=errors.jurisdiction, classes=['control-medium']) }} - -{{ form.input('temporal_coverage', label=_('Temporal Coverage'), id='field-temporal_coverage', -placeholder=_('The timespan the dataset relates to: Current, 2008, 2008-2012, Jan 2012'), -value=data.temporal_coverage, error=errors.temporal_coverage, classes=['control-medium']) }} - -{{ form.select('data_state', label=_('Data State'), options= [{'value': 'active', 'text': 'Active'}, {'value': 'inactive', 'text': 'Inactive'}], -selected='Active', error=errors.data_state) }} - -{{ form.input('update_freq', label=_('Update Frequency'), id='field-update_freq', -placeholder=_('How often the dataset is updated. Eg: Daily, Weekly, Never'), -value=data.update_freq, error=errors.update_freq, classes=['control-medium']) }} - -{% endblock %} {% block package_basic_fields_license %} @@ -61,3 +23,4 @@ {% endblock %} + --- a/ckanext/datagovau/templates/package/snippets/package_metadata_fields.html +++ b/ckanext/datagovau/templates/package/snippets/package_metadata_fields.html @@ -8,20 +8,6 @@ {% endblock %} {# https://github.com/okfn/ckan/blob/master/ckan/templates/macros/form.html documents the macros for fields #} - -{% block package_metadata_fields %} -{{ form.input('url', label=_('Source URL'), id='field-url', -placeholder=_('URL where dataset came from or more information can be obtained'), -value=data.url, error=errors.url, classes=['control-medium']) }} -{{ super() }} - -{% endblock %} - -{% block package_metadata_author %} -{% endblock %} - -{% block package_metadata_fields_maintainer %} -{% endblock %} {% block dataset_fields %} {% if data.groups %} @@ -41,3 +27,4 @@ {% set group_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/group/autocomplete?q=?', 'data-module-key': 'id', 'data-module-label': 'title'} %} {{ form.input(group_name, label=_('Add Group/Theme'), id="field-group", value=data[group_name], classes=['control-medium'], attrs=group_attrs) }} {% endblock %} + --- a/ckanext/datagovau/templates/package/snippets/resource_item.html +++ /dev/null @@ -1,54 +1,1 @@ -{% ckan_extends %} -{% set url = h.url_for(controller='package', action='resource_read', id=pkg.name, resource_id=res.id) %} -
  • - {% block resource_item_title %} - - {{ h.resource_display_name(res) | truncate(50) }}{{ res.format }} - {{ h.popular('views', res.tracking_summary.total, min=10) }} - - {% endblock %} -

    - {% if res.description %} - {{ h.markdown_extract(res.description, extract_length=80) }} - {% else %} - {{ _('No description for this resource') }} - {% endif %} -

    - {% block resource_item_explore %} -
    - - - {{ _('Explore') }} - - - -
    - {% endblock %} -