This CKAN Extension customises a CKAN instance for the hosting of data.gov.au. | This CKAN Extension customises a CKAN instance for the hosting of data.gov.au. |
It comprises: | It comprises: |
* A CKAN Extension "plugin" at ``ckanext/datagovau/plugin.py`` which, when | * A CKAN Extension "plugin" at ``ckanext/datagovau/plugin.py`` which, when |
loaded, overrides various settings in the core ``ini``-file to provide: | loaded, overrides various settings in the core ``ini``-file to provide: |
* A path to local customisations of the core templates | * A path to local customisations of the core templates to include AGLS/Dublin Core minimum metadata |
* A custom Package edit form that defaults to cc-by licence | * A custom Package edit form that defaults to cc-by licence |
* A custom n3/rdf output format | * A custom n3/rdf output format |
* Replaces links with http/https protocol independent versions | |
* Provides HTML to users to embed data previews on their own website | |
* A cut down licenses.json file | * A cut down licenses.json file |
Installation | Installation |
============ | ============ |
To install this package, from your CKAN virtualenv, run the following from your CKAN base folder (e.g. ``pyenv/``):: | To install this package, from your CKAN virtualenv, run the following from your CKAN base folder (e.g. ``pyenv/``):: |
pip install -e git+https://github.com/okfn/ckanext-datagovau#egg=ckanext-datagovau | pip install -e git+https://github.com/okfn/ckanext-datagovau#egg=ckanext-datagovau |
Then activate it by setting ``ckan.plugins = datagovau`` in your main ``ini``-file. | Then activate it by setting ``ckan.plugins = datagovau`` in your main ``ini``-file. |
To add the cut down licenses.json set ``licenses_group_url = http://%(ckan.site_url)/licenses.json`` | To add the cut down licenses.json set ``licenses_group_url = http://%(ckan.site_url)/licenses.json`` |
or copy ``ckanext/datagovau/theme/public/licenses.json`` to the same folder as your CKAN config ini file | or copy ``ckanext/datagovau/theme/public/licenses.json`` to the same folder as your CKAN config ini file |
and set ``licenses_group_url = file://%(here)s/licenses.json`` | and set ``licenses_group_url = file://%(here)s/licenses.json`` |
# This is a basic VCL configuration file for varnish. See the vcl(7) | # This is a basic VCL configuration file for varnish. See the vcl(7) |
# man page for details on VCL syntax and semantics. | # man page for details on VCL syntax and semantics. |
# | # |
# Default backend definition. Set this to point to your content | # Default backend definition. Set this to point to your content |
# server. | # server. |
# | # |
backend default { | backend default { |
.host = "127.0.0.1"; | .host = "127.0.0.1"; |
.port = "8080"; | .port = "8080"; |
} | } |
backend geoserver { | |
.host = "172.31.18.207"; | |
.port = "8983"; | |
} | |
sub vcl_fetch { | sub vcl_fetch { |
set beresp.grace = 1h; | set beresp.grace = 1h; |
unset beresp.http.Server; | |
if (beresp.http.content-type ~ "(text|application)") { | # These status codes should always pass through and never cache. |
if ( beresp.status >= 500 ) { | |
set beresp.ttl = 0s; | |
} | |
if (beresp.http.content-type ~ "(text|javascript|json|xml|html)") { | |
set beresp.do_gzip = true; | set beresp.do_gzip = true; |
} | } |
# CKAN cache headers are used by Varnish cache, but should not be propagated to | |
# the Internet. Tell browsers and proxies not to cache. This means Varnish always | |
# gets the responsibility to server the right content at all times. | |
if (beresp.http.Cache-Control ~ "max-age") { | |
unset beresp.http.set-cookie; | |
set beresp.http.Cache-Control = "no-cache"; | |
} | |
# Encourage assets to be cached by proxies and browsers | |
# JS and CSS may be gzipped depending on headers | |
# see https://developers.google.com/speed/docs/best-practices/caching | |
if (req.url ~ "\.(css|js)") { | |
set beresp.http.Vary = "Accept-Encoding"; | |
} | |
# Encourage assets to be cached by proxies and browsers for 1 day | |
if (req.url ~ "\.(png|gif|jpg|swf|css|js)") { | |
unset beresp.http.set-cookie; | |
set beresp.http.Cache-Control = "public, max-age=86400"; | |
set beresp.ttl = 1d; | |
} | |
# Encourage CKAN vendor assets (which are versioned) to be cached by | |
# by proxies and browsers for 1 year | |
if (req.url ~ "^/scripts/vendor/") { | |
unset beresp.http.set-cookie; | |
set beresp.http.Cache-Control = "public, max-age=31536000"; | |
set beresp.ttl = 12m; | |
} | |
} | |
sub vcl_recv { | |
if (req.http.user-agent ~ "Ezooms" || req.http.user-agent ~ "Ahrefs") { | |
error 403; | |
} | |
if (req.url ~ "^/geoserver/") { | |
set req.backend = geoserver; | |
} else { | |
set req.backend = default; | |
#redirect secure traffic to https | |
if ( (req.http.Cookie ~ "auth_tkt" || req.http.Cookie ~ "ckan" || req.url ~ "user/(reset|login)") && req.http.X-Forwarded-Proto !~ "(?i)https") { | |
set req.http.x-Redir-Url = "https://data.gov.au" + req.url; | |
error 753 req.http.x-Redir-Url; | |
} | |
# remove locale links | |
if (req.url ~ "/((?!js)..|.._..|sr_Latn)/") { | |
set req.http.x-Redir-Url = regsub(req.url, "/((?!js)..|.._..|sr_Latn)/", "/"); | |
error 751 req.http.x-Redir-Url; | |
} | |
# rewrite broken resources | |
if (req.url ~ "leaflet") { | |
set req.url = regsub(req.url, "fanstatic/ckanext-spatial/:version:2013-09-13T02:32:17.87/:bundle:js/vendor/leaflet/images", "js/vendor/leaflet/images"); | |
} | |
# remove old hostnames | |
if (req.http.host ~ "data.australia.gov.au") { | |
set req.http.x-Redir-Url = "http://data.gov.au" + req.url; | |
error 751 req.http.x-Redir-Url; | |
} | |
if (req.url ~ "^/_tracking") { | |
// exclude web spiders from statistics | |
if (req.http.user-agent ~ "Googlebot" || req.http.user-agent ~ "baidu" || req.http.user-agent ~ "bing") { | |
error 200; | |
} else { | |
return (pass); | |
} | |
} | |
if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { | if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { |
unset beresp.http.set-cookie; | //Varnish to deliver content from cache even if the request othervise indicates that the request should be passed |
return(lookup); | |
} | } |
} | } |
sub vcl_recv { | // Remove has_js and Google Analytics cookies. Evan added sharethis cookies |
if (req.url ~ "^/_tracking") { | set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|has_js|cookie-agreed-en|_csoot|_csuid|_chartbeat2)=[^;]*", ""); |
return (pass); | |
} | // Remove a ";" prefix, if present. |
if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { | set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", ""); |
return(lookup); | // Remove empty cookies. |
} | if (req.http.Cookie ~ "^\s*$") { |
if (req.url ~ "/(..|.._..)/") { | unset req.http.Cookie; |
set req.url = regsub(req.url, "/((?!js)..|.._..)/", "/"); | } |
} | |
if (req.http.Cookie) { | remove req.http.X-Forwarded-For; |
set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing | set req.http.X-Forwarded-For = req.http.X-Real-IP; |
if (req.http.Cookie == "") { | |
remove req.http.Cookie; | |
} | |
} | |
} | } |
sub vcl_hash { | sub vcl_hash { |
# http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl | # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl |
set req.url = regsub(req.url, "(?:(.com|.au))/((?!js)..|.._..)/", "/"); | set req.url = regsub(req.url, "(?:(.com|.au))/((?!js)..|.._..|sr_Latn)/", "/"); |
hash_data(req.url); | hash_data(req.url); |
if (req.http.host) { | if (req.http.host) { |
hash_data(req.http.host); | hash_data(req.http.host); |
} else { | } else { |
hash_data(server.ip); | hash_data(server.ip); |
} | } |
return (hash); | if (req.http.Cookie) { |
hash_data(req.http.Cookie); | |
} | |
} | } |
sub vcl_deliver { | sub vcl_deliver { |
if (!resp.http.Vary) { | if (!resp.http.Vary) { |
set resp.http.Vary = "Accept-Encoding"; | set resp.http.Vary = "Accept-Encoding"; |
} else if (resp.http.Vary !~ "(?i)Accept-Encoding") { | } else if (resp.http.Vary !~ "(?i)Accept-Encoding") { |
set resp.http.Vary = resp.http.Vary + ",Accept-Encoding"; | set resp.http.Vary = resp.http.Vary + ",Accept-Encoding"; |
} | } |
if (obj.hits > 0) { | remove resp.http.X-Varnish; |
set resp.http.X-Cache = "HIT"; | remove resp.http.Via; |
} else { | remove resp.http.Age; |
set resp.http.X-Cache = "MISS"; | remove resp.http.X-Powered-By; |
} | if (req.url ~ "^/geoserver/") { |
set resp.http.Access-Control-Allow-Origin = "*"; | |
set resp.http.Access-Control-Allow-Methods = "GET, POST, PUT, DELETE"; | |
set resp.http.Access-Control-Allow-Headers = "Origin, X-Requested-With, Content-Type, Accept"; | |
} | |
} | } |
sub vcl_error { | |
remove obj.http.Server; | |
if (obj.status == 751) { | |
set obj.http.Location = obj.response; | |
set obj.status = 301; | |
return (deliver); | |
} | |
if (obj.status == 753) { | |
set obj.http.Location = obj.response; | |
set obj.status = 301; | |
return (deliver); | |
} | |
} | |
# | # |
# Below is a commented-out copy of the default VCL logic. If you | # Below is a commented-out copy of the default VCL logic. If you |
# redefine any of these subroutines, the built-in logic will be | # redefine any of these subroutines, the built-in logic will be |
# appended to your code. | # appended to your code. |
# sub vcl_recv { | # sub vcl_recv { |
# if (req.restarts == 0) { | # if (req.restarts == 0) { |
# if (req.http.x-forwarded-for) { | # if (req.http.x-forwarded-for) { |
# set req.http.X-Forwarded-For = | # set req.http.X-Forwarded-For = |
# req.http.X-Forwarded-For + ", " + client.ip; | # req.http.X-Forwarded-For + ", " + client.ip; |
# } else { | # } else { |
# set req.http.X-Forwarded-For = client.ip; | # set req.http.X-Forwarded-For = client.ip; |
# } | # } |
# } | # } |
# if (req.request != "GET" && | # if (req.request != "GET" && |
# req.request != "HEAD" && | # req.request != "HEAD" && |
# req.request != "PUT" && | # req.request != "PUT" && |
# req.request != "POST" && | # req.request != "POST" && |
# req.request != "TRACE" && | # req.request != "TRACE" && |
# req.request != "OPTIONS" && | # req.request != "OPTIONS" && |
# req.request != "DELETE") { | # req.request != "DELETE") { |
# /* Non-RFC2616 or CONNECT which is weird. */ | # /* Non-RFC2616 or CONNECT which is weird. */ |
# return (pipe); | # return (pipe); |
# } | # } |
# if (req.request != "GET" && req.request != "HEAD") { | # if (req.request != "GET" && req.request != "HEAD") { |
# /* We only deal with GET and HEAD by default */ | # /* We only deal with GET and HEAD by default */ |
# return (pass); | # return (pass); |
# } | # } |
# if (req.http.Authorization || req.http.Cookie) { | # if (req.http.Authorization || req.http.Cookie) { |
# /* Not cacheable by default */ | # /* Not cacheable by default */ |
# return (pass); | # return (pass); |
# } | # } |
# return (lookup); | # return (lookup); |
# } | # } |
# | # |
# sub vcl_pipe { | # sub vcl_pipe { |
# # Note that only the first request to the backend will have | # # Note that only the first request to the backend will have |
# # X-Forwarded-For set. If you use X-Forwarded-For and want to | # # X-Forwarded-For set. If you use X-Forwarded-For and want to |
# # have it set for all requests, make sure to have: | # # have it set for all requests, make sure to have: |
# # set bereq.http.connection = "close"; | # # set bereq.http.connection = "close"; |
# # here. It is not set by default as it might break some broken web | # # here. It is not set by default as it might break some broken web |
# # applications, like IIS with NTLM authentication. | # # applications, like IIS with NTLM authentication. |
# return (pipe); | # return (pipe); |
# } | # } |
# | # |
# sub vcl_pass { | # sub vcl_pass { |
# return (pass); | # return (pass); |
# } | # } |
# | # |
# sub vcl_hash { | # sub vcl_hash { |
# hash_data(req.url); | # hash_data(req.url); |
# if (req.http.host) { | # if (req.http.host) { |
# hash_data(req.http.host); | # hash_data(req.http.host); |
# } else { | # } else { |
# hash_data(server.ip); | # hash_data(server.ip); |
# } | # } |
# return (hash); | # return (hash); |
# } | # } |
# | # |
# sub vcl_hit { | # sub vcl_hit { |
# return (deliver); | # return (deliver); |
# } | # } |
# | # |
# sub vcl_miss { | # sub vcl_miss { |
# return (fetch); | # return (fetch); |
# } | # } |
# | # |
# sub vcl_fetch { | # sub vcl_fetch { |
# if (beresp.ttl <= 0s || | # if (beresp.ttl <= 0s || |
# beresp.http.Set-Cookie || | # beresp.http.Set-Cookie || |
# beresp.http.Vary == "*") { | # beresp.http.Vary == "*") { |
# /* | # /* |
# * Mark as "Hit-For-Pass" for the next 2 minutes | # * Mark as "Hit-For-Pass" for the next 2 minutes |
# */ | # */ |
# set beresp.ttl = 120 s; | # set beresp.ttl = 120 s; |
# return (hit_for_pass); | # return (hit_for_pass); |
# } | # } |
# return (deliver); | # return (deliver); |
# } | # } |
# | # |
# sub vcl_deliver { | # sub vcl_deliver { |
# return (deliver); | # return (deliver); |
# } | # } |
# | # |
# sub vcl_error { | # sub vcl_error { |
# set obj.http.Content-Type = "text/html; charset=utf-8"; | # set obj.http.Content-Type = "text/html; charset=utf-8"; |
# set obj.http.Retry-After = "5"; | # set obj.http.Retry-After = "5"; |
# synthetic {" | # synthetic {" |
# <?xml version="1.0" encoding="utf-8"?> | # <?xml version="1.0" encoding="utf-8"?> |
# <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" | # <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" |
# "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> | # "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> |
# <html> | # <html> |
# <head> | # <head> |
# <title>"} + obj.status + " " + obj.response + {"</title> | # <title>"} + obj.status + " " + obj.response + {"</title> |
# </head> | # </head> |
# <body> | # <body> |
# <h1>Error "} + obj.status + " " + obj.response + {"</h1> | # <h1>Error "} + obj.status + " " + obj.response + {"</h1> |
# <p>"} + obj.response + {"</p> | # <p>"} + obj.response + {"</p> |
# <h3>Guru Meditation:</h3> | # <h3>Guru Meditation:</h3> |
# <p>XID: "} + req.xid + {"</p> | # <p>XID: "} + req.xid + {"</p> |
# <hr> | # <hr> |
# <p>Varnish cache server</p> | # <p>Varnish cache server</p> |
# </body> | # </body> |
# </html> | # </html> |
# "}; | # "}; |
# return (deliver); | # return (deliver); |
# } | # } |
# | # |
# sub vcl_init { | # sub vcl_init { |
# return (ok); | # return (ok); |
# } | # } |
# | # |
# sub vcl_fini { | # sub vcl_fini { |
# return (ok); | # return (ok); |
# } | # } |
paster --plugin=ckan db clean --config=development.ini | |
echo "drop extension postgis cascade;" | psql -d ckantest | |
paster --plugin=ckan db clean --config=development.ini | |
#to initiate for first time instead of load from dump | |
#paster --plugin=ckan db init --config=development.in | |
#paster --plugin=ckan user add maxious password=snmc email=maxious@gmail.com | |
#paster --plugin=ckan sysadmin add maxious | |
#paster --plugin=ckan db dump dump.db | |
#paster --plugin=ckan db load --config=development.ini dump.db | |
paster --plugin=ckan db load --config=development.ini dump.harvest.db | |
echo "create extension postgis;" | psql -d ckantest | |
#sleep 2 | |
paster --plugin=ckan search-index rebuild --config=development.ini | |
#rm -r /tmp/pairtree_* | |
import urllib | |
import json | |
from pprint import pprint | |
import logging | |
import ckan.logic as logic | |
import hashlib | |
import threading | |
from ckan.common import _, c, request, response | |
from pylons import config | |
from webob.multidict import UnicodeMultiDict | |
from paste.util.multidict import MultiDict | |
log = logging.getLogger(__name__) | |
from ckan.controllers.api import ApiController | |
class DGAApiController(ApiController): | |
def _post_analytics(self,user,request_obj_type,request_function,request_id): | |
if (config.get('googleanalytics.id') != None): | |
data = urllib.urlencode({ | |
"v":1, | |
"tid":config.get('googleanalytics.id'), | |
"cid":hashlib.md5(user).hexdigest(), | |
"t":"event", | |
"dh":c.environ['HTTP_HOST'], | |
"dp":c.environ['PATH_INFO'], | |
"dr":c.environ.get('HTTP_REFERER',''), | |
"ec":"CKAN API Request", | |
"ea":request_obj_type+request_function, | |
"el":request_id, | |
}) | |
log.debug("Sending API Analytics Data: "+data) | |
# send analytics asynchronously | |
threading.Thread(target=urllib.urlopen,args=("http://www.google-analytics.com/collect", data)).start() | |
def action(self, logic_function, ver=None): | |
try: | |
function = logic.get_action(logic_function) | |
except Exception,e: | |
log.debug(e) | |
pass | |
try: | |
side_effect_free = getattr(function, 'side_effect_free', False) | |
request_data = self._get_request_data(try_url_params=side_effect_free) | |
if isinstance(request_data, dict): | |
id = request_data.get('id','') | |
if 'q' in request_data.keys(): | |
id = request_data['q'] | |
if 'query' in request_data.keys(): | |
id = request_data['query'] | |
self._post_analytics(c.user,logic_function,'', id) | |
except Exception,e: | |
print log.debug(e) | |
pass | |
return ApiController.action(self,logic_function, ver) | |
def list(self, ver=None, register=None, subregister=None, id=None): | |
self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"list",id) | |
return ApiController.list(self,ver, register, subregister, id) | |
def show(self, ver=None, register=None, subregister=None, id=None, id2=None): | |
self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"show",id) | |
return ApiController.show(self,ver, register, subregister, id,id2) | |
def update(self, ver=None, register=None, subregister=None, id=None, id2=None): | |
self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"update",id) | |
return ApiController.update(self,ver, register, subregister, id,id2) | |
def delete(self, ver=None, register=None, subregister=None, id=None, id2=None): | |
self._post_analytics(c.user,register+("_"+str(subregister) if subregister else ""),"delete",id) | |
return ApiController.delete(self,ver, register, subregister, id,id2) | |
def search(self, ver=None, register=None): | |
id = None | |
try: | |
params = MultiDict(self._get_search_params(request.params)) | |
if 'q' in params.keys(): | |
id = params['q'] | |
if 'query' in params.keys(): | |
id = params['query'] | |
except ValueError, e: | |
print str(e) | |
pass | |
self._post_analytics(c.user,register,"search",id) | |
import logging | import logging |
import ckan.plugins as plugins | import ckan.plugins as plugins |
import ckan.lib as lib | import ckan.lib as lib |
import ckan.lib.dictization.model_dictize as model_dictize | import ckan.lib.dictization.model_dictize as model_dictize |
import ckan.plugins.toolkit as tk | import ckan.plugins.toolkit as tk |
import ckan.model as model | import ckan.model as model |
from pylons import config | from pylons import config |
from routes.mapper import SubMapper, Mapper as _Mapper | |
from sqlalchemy import orm | |
import ckan.model | |
#parse the activity feed for last active non-system user | #parse the activity feed for last active non-system user |
def get_last_active_user(id): | def get_last_active_user(id): |
system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')}) | system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')}) |
user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']] | user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']] |
user = None | user = None |
if len(user_list) > 0: | if len(user_list) > 0: |
user = user_list[0].get('user_id', None) | user = user_list[0].get('user_id', None) |
if user is None: | if user is None: |
return system_user | return system_user |
else: | else: |
return lib.helpers.get_action('user_show',{'id':user}) | return lib.helpers.get_action('user_show',{'id':user}) |
# get user created datasets and those they have edited | # get user created datasets and those they have edited |
def get_user_datasets(user_dict): | def get_user_datasets(user_dict): |
created_datasets_list = user_dict['datasets'] | created_datasets_list = user_dict['datasets'] |
active_datasets_list = [x['data']['package'] for x in | active_datasets_list = [x['data']['package'] for x in |
lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] | lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] |
return created_datasets_list + active_datasets_list | raw_list = created_datasets_list + active_datasets_list |
filtered_dict = {} | |
for dataset in raw_list: | |
if dataset['id'] not in filtered_dict.keys(): | |
filtered_dict[dataset['id']] = dataset | |
return filtered_dict.values() | |
class DataGovAuPlugin(plugins.SingletonPlugin, | class DataGovAuPlugin(plugins.SingletonPlugin, |
tk.DefaultDatasetForm): | tk.DefaultDatasetForm): |
'''An example IDatasetForm CKAN plugin. | '''An example IDatasetForm CKAN plugin. |
Uses a tag vocabulary to add a custom metadata field to datasets. | Uses a tag vocabulary to add a custom metadata field to datasets. |
''' | ''' |
plugins.implements(plugins.IConfigurer, inherit=False) | plugins.implements(plugins.IConfigurer, inherit=False) |
plugins.implements(plugins.IDatasetForm, inherit=False) | plugins.implements(plugins.IDatasetForm, inherit=False) |
plugins.implements(plugins.ITemplateHelpers, inherit=False) | plugins.implements(plugins.ITemplateHelpers, inherit=False) |
plugins.implements(plugins.IRoutes, inherit=True) | |
def before_map(self, map): | |
# Helpers to reduce code clutter | |
GET = dict(method=['GET']) | |
PUT = dict(method=['PUT']) | |
POST = dict(method=['POST']) | |
DELETE = dict(method=['DELETE']) | |
GET_POST = dict(method=['GET', 'POST']) | |
# intercept API calls that we want to capture analytics on | |
register_list = [ | |
'package', | |
'dataset', | |
'resource', | |
'tag', | |
'group', | |
'related', | |
'revision', | |
'licenses', | |
'rating', | |
'user', | |
'activity' | |
] | |
register_list_str = '|'.join(register_list) | |
# /api ver 3 or none | |
with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/3|}', | |
ver='/3') as m: | |
m.connect('/action/{logic_function}', action='action', | |
conditions=GET_POST) | |
# /api ver 1, 2, 3 or none | |
with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1|/2|/3|}', | |
ver='/1') as m: | |
m.connect('/search/{register}', action='search') | |
# /api/rest ver 1, 2 or none | |
with SubMapper(map, controller='ckanext.datagovau.controller:DGAApiController', path_prefix='/api{ver:/1|/2|}', | |
ver='/1', requirements=dict(register=register_list_str) | |
) as m: | |
m.connect('/rest/{register}', action='list', conditions=GET) | |
m.connect('/rest/{register}', action='create', conditions=POST) | |
m.connect('/rest/{register}/{id}', action='show', conditions=GET) | |
m.connect('/rest/{register}/{id}', action='update', conditions=PUT) | |
m.connect('/rest/{register}/{id}', action='update', conditions=POST) | |
m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE) | |
return map | |
def update_config(self, config): | def update_config(self, config): |
# Add this plugin's templates dir to CKAN's extra_template_paths, so | # Add this plugin's templates dir to CKAN's extra_template_paths, so |
# that CKAN will use this plugin's custom templates. | # that CKAN will use this plugin's custom templates. |
# here = os.path.dirname(__file__) | # here = os.path.dirname(__file__) |
# rootdir = os.path.dirname(os.path.dirname(here)) | # rootdir = os.path.dirname(os.path.dirname(here)) |
tk.add_template_directory(config, 'templates') | tk.add_template_directory(config, 'templates') |
tk.add_public_directory(config, 'theme/public') | tk.add_public_directory(config, 'theme/public') |
tk.add_resource('theme/public', 'ckanext-datagovau') | tk.add_resource('theme/public', 'ckanext-datagovau') |
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' | # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' |
def get_helpers(self): | def get_helpers(self): |
return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets} | return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets} |
def is_fallback(self): | def is_fallback(self): |
# Return True to register this plugin as the default handler for | # Return True to register this plugin as the default handler for |
# package types not handled by any other IDatasetForm plugin. | # package types not handled by any other IDatasetForm plugin. |
return True | return True |
def package_types(self): | def package_types(self): |
# This plugin doesn't handle any special package types, it just | # This plugin doesn't handle any special package types, it just |
# registers itself as the default (above). | # registers itself as the default (above). |
return [] | return [] |
def create_package_schema(self): | def create_package_schema(self): |
schema = super(DataGovAuPlugin, self).create_package_schema() | schema = super(DataGovAuPlugin, self).create_package_schema() |
schema = self._modify_package_schema(schema) | schema = self._modify_package_schema(schema) |
return schema | return schema |
def update_package_schema(self): | def update_package_schema(self): |
schema = super(DataGovAuPlugin, self).update_package_schema() | schema = super(DataGovAuPlugin, self).update_package_schema() |
schema = self._modify_package_schema(schema) | schema = self._modify_package_schema(schema) |
return schema | return schema |
def show_package_schema(self): | def show_package_schema(self): |
schema = super(DataGovAuPlugin, self).show_package_schema() | schema = super(DataGovAuPlugin, self).show_package_schema() |
# Don't show vocab tags mixed in with normal 'free' tags | # Don't show vocab tags mixed in with normal 'free' tags |
# (e.g. on dataset pages, or on the search page) | # (e.g. on dataset pages, or on the search page) |
schema['tags']['__extras'].append(tk.get_converter('free_tags_only')) | schema['tags']['__extras'].append(tk.get_converter('free_tags_only')) |
# Add our custom_text field to the dataset schema. | # Add our custom_text field to the dataset schema. |
# ignore_missing == optional | # ignore_missing == optional |
# ignore_empty == mandatory but not for viewing | # ignore_empty == mandatory but not for viewing |
# !!! always convert_from_extras first | # !!! always convert_from_extras first |
schema.update({ | schema.update({ |
'agency_program': [tk.get_converter('convert_from_extras'), | 'agency_program': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_missing')], | tk.get_validator('ignore_missing')], |
'contact_point': [tk.get_converter('convert_from_extras'), | 'contact_point': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'spatial_coverage': [tk.get_converter('convert_from_extras'), | 'spatial_coverage': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'granularity': [tk.get_converter('convert_from_extras'), | 'granularity': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'jurisdiction': [tk.get_converter('convert_from_extras'), | 'jurisdiction': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'temporal_coverage': [tk.get_converter('convert_from_extras'), | 'temporal_coverage': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'data_state': [tk.get_converter('convert_from_extras'), | 'data_state': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')], | tk.get_validator('ignore_empty')], |
'update_freq': [tk.get_converter('convert_from_extras'), | 'update_freq': [tk.get_converter('convert_from_extras'), |
tk.get_validator('ignore_empty')] | tk.get_validator('ignore_empty')] |
}) | }) |
return schema | return schema |
def _modify_package_schema(self, schema): | def _modify_package_schema(self, schema): |
# Add our custom_test metadata field to the schema, this one will use | # Add our custom_test metadata field to the schema, this one will use |
# convert_to_extras instead of convert_to_tags. | # convert_to_extras instead of convert_to_tags. |
# ignore_missing == optional | # ignore_missing == optional |
# not_empty == mandatory, enforced here while modifying | # not_empty == mandatory, enforced here while modifying |
schema.update({ | schema.update({ |
'agency_program': [tk.get_validator('ignore_missing'), | 'agency_program': [tk.get_validator('ignore_missing'), |
tk.get_converter('convert_to_extras')], | tk.get_converter('convert_to_extras')], |
'contact_point': [tk.get_converter('convert_to_extras'), | 'contact_point': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'spatial_coverage': [tk.get_converter('convert_to_extras'), | 'spatial_coverage': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'granularity': [tk.get_converter('convert_to_extras'), | 'granularity': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'jurisdiction': [tk.get_converter('convert_to_extras'), | 'jurisdiction': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'temporal_coverage': [tk.get_converter('convert_to_extras'), | 'temporal_coverage': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'data_state': [tk.get_converter('convert_to_extras'), | 'data_state': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')], | tk.get_validator('not_empty')], |
'update_freq': [tk.get_converter('convert_to_extras'), | 'update_freq': [tk.get_converter('convert_to_extras'), |
tk.get_validator('not_empty')] | tk.get_validator('not_empty')] |
}) | }) |
return schema | return schema |
# These methods just record how many times they're called, for testing | # These methods just record how many times they're called, for testing |
# purposes. | # purposes. |
# TODO: It might be better to test that custom templates returned by | # TODO: It might be better to test that custom templates returned by |
# these methods are actually used, not just that the methods get | # these methods are actually used, not just that the methods get |
# called. | # called. |
def setup_template_variables(self, context, data_dict): | def setup_template_variables(self, context, data_dict): |
return super(DataGovAuPlugin, self).setup_template_variables( | return super(DataGovAuPlugin, self).setup_template_variables( |
context, data_dict) | context, data_dict) |
def new_template(self): | def new_template(self): |
return super(DataGovAuPlugin, self).new_template() | return super(DataGovAuPlugin, self).new_template() |
def read_template(self): | def read_template(self): |
return super(DataGovAuPlugin, self).read_template() | return super(DataGovAuPlugin, self).read_template() |
def edit_template(self): | def edit_template(self): |
return super(DataGovAuPlugin, self).edit_template() | return super(DataGovAuPlugin, self).edit_template() |
def search_template(self): | def search_template(self): |
return super(DataGovAuPlugin, self).search_template() | return super(DataGovAuPlugin, self).search_template() |
def history_template(self): | def history_template(self): |
return super(DataGovAuPlugin, self).history_template() | return super(DataGovAuPlugin, self).history_template() |
def package_form(self): | def package_form(self): |
return super(DataGovAuPlugin, self).package_form() | return super(DataGovAuPlugin, self).package_form() |
{% ckan_extends %} | |
{% block meta %} | |
{{ super() }} | |
<meta name="google-site-verification" content="B8scmW2_LMm4IW_ogXK9lojznO57GBpYlWrFjfGz3X8" /> | |
{% endblock %} | |
{% ckan_extends %} | |
{% block scripts %} | |
{{ super() }} | |
<script> | |
preload_resource.url = preload_resource.url.replace("https:","").replace("http:",""); | |
preload_resource.original_url = preload_resource.original_url.replace("https:","").replace("http:",""); | |
</script> | |
{% endblock %} | |
<div class="module-content ckanext-datapreview"> | |
{% if embed %} | |
{# images can be embedded directly #} | |
<img src="{{ resource_url }}"></img> | |
{% else %} | |
<div class="data-viewer-error js-hide"> | |
<p class="text-error"> | |
<i class="icon-info-sign"></i> | |
{{ _('This resource can not be previewed at the moment.') }} | |
<a href="#" data-toggle="collapse" data-target="#data-view-error"> | |
{{ _('Click here for more information.') }} | |
</a> | |
</p> | |
<p id="data-view-error" class="collapse"></p> | |
<p> | |
<a href="{{ raw_resource_url }}" class="btn btn-large resource-url-analytics" target="_blank"> | |
<i class="icon-large icon-download"></i> | |
{{ _('Download resource') }} | |
</a> | |
</p> | |
</div> | |
<iframe src="{{ resource_url }}" frameborder="0" width="100%" data-module="data-viewer"> | |
<p>{{ _('Your browser does not support iframes.') }}</p> | |
</iframe> | |
{% endif %} | |
</div> | |
<div class="embedhint" style="padding-left: 15px;"> | |
<h4>Embed this visualisation in your own website...</h4> | |
Copy the HTML in the box below and you can display this visualisation on your own website.<br/> | |
<textarea style="width:97%"><iframe src="{{ resource_url }}" frameborder="0" width="100%" height="600px"></iframe></textarea> | |
</div> | |
{% ckan_extends %} | {% ckan_extends %} |
{% block primary_content_inner %} | {% block primary_content_inner %} |
{{ super() }} | {{ super() }} |
<div class="module-content"> | <div class="module-content"> |
{{ h.disqus_comments() }} | {{ h.disqus_comments() }} |
</div> | </div> |
{% endblock %} | {% endblock %} |
{% block package_additional_info %} | {% block package_additional_info %} |
<section class="additional-info" prefix="dc: http://purl.org/dc/elements/1.1/ dcat: http://www.w3.org/ns/dcat# | <section class="additional-info" prefix="dc: http://purl.org/dc/elements/1.1/ dcat: http://www.w3.org/ns/dcat# |
dcam: http://purl.org/dc/dcam/ | dcam: http://purl.org/dc/dcam/ |
aglsterms: http://www.agls.gov.au/agls/terms/ | aglsterms: http://www.agls.gov.au/agls/terms/ |
agentterms: http://www.agls.gov.au/agls/agentterms/ | agentterms: http://www.agls.gov.au/agls/agentterms/ |
availterms: http://www.agls.gov.au/agls/availterms/ | availterms: http://www.agls.gov.au/agls/availterms/ |
adminterms: http://www.agls.gov.au/agls/adminterms/ | adminterms: http://www.agls.gov.au/agls/adminterms/ |
dct: http://purl.org/dc/terms/"> | dct: http://purl.org/dc/terms/"> |
<h3>{{ _('Additional Info') }}</h3> | <h3>{{ _('Additional Info') }}</h3> |
<table class="table table-striped table-bordered table-condensed"> | <table class="table table-striped table-bordered table-condensed"> |
<thead> | <thead> |
<tr> | <tr> |
<th scope="col">{{ _('Field') }}</th> | <th scope="col">{{ _('Field') }}</th> |
<th scope="col">{{ _('Value') }}</th> | <th scope="col">{{ _('Value') }}</th> |
</tr> | </tr> |
</thead> | </thead> |
<tbody> | <tbody> |
{# Add our custom field to the dataset read page. #} | {# Add our custom field to the dataset read page. #} |
{% if pkg.get('metadata_created') %} | {% if pkg.get('metadata_created') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Date Published</th> | <th scope="row" class="dataset-label">Date Published</th> |
<td class="dataset-details" property="dct:issued">{{ pkg.metadata_created.split("T")[0] }}</td> | <td class="dataset-details" property="dct:issued">{{ pkg.metadata_created.split("T")[0] }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('metadata_updated') %} | {% if pkg.get('metadata_updated') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Date Updated</th> | <th scope="row" class="dataset-label">Date Updated</th> |
<td class="dataset-details" property="dct:updated">{{ pkg.metadata_updated.split("T")[0] }}</td> | <td class="dataset-details" property="dct:updated">{{ pkg.metadata_updated.split("T")[0] }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('agency_program') %} | {% if pkg.get('agency_program') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Agency Program</th> | <th scope="row" class="dataset-label">Agency Program</th> |
<td class="dataset-details"> {{ pkg.agency_program }}</td> | <td class="dataset-details"> {{ pkg.agency_program }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('url') %} | {% if pkg.get('url') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">{{ _('Source') }}</th> | <th scope="row" class="dataset-label">{{ _('Source') }}</th> |
<td class="dataset-details" property="dct:source">{{ h.link_to(pkg.get('url'), | <td class="dataset-details" property="dct:source">{{ h.link_to(pkg.get('url'), |
pkg.get('url'), rel='dct:source', target='_blank') }} | pkg.get('url'), rel='dct:source', target='_blank') }} |
</td> | </td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('contact_point') %} | {% if pkg.get('contact_point') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Contact Point</th> | <th scope="row" class="dataset-label">Contact Point</th> |
<td class="dataset-details" property="dcat:contactPoint"> {{ h.mail_to(email_address=pkg.contact_point, | <td class="dataset-details" property="dcat:contactPoint"> {{ h.mail_to(email_address=pkg.contact_point, |
name=pkg.contact_point) }}</td> | name=pkg.contact_point) }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('spatial_coverage') %} | {% if pkg.get('spatial_coverage') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Geospatial Coverage</th> | <th scope="row" class="dataset-label">Geospatial Coverage</th> |
<td class="dataset-details" property="dct:spatial"> {{ pkg.spatial_coverage }}</td> | <td class="dataset-details" property="dct:spatial"> {{ pkg.spatial_coverage }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('granularity') %} | {% if pkg.get('granularity') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Data Granularity</th> | <th scope="row" class="dataset-label">Data Granularity</th> |
<td class="dataset-details"> {{ pkg.granularity }}</td> | <td class="dataset-details"> {{ pkg.granularity }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('jurisdiction') %} | {% if pkg.get('jurisdiction') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Government Jurisdiction</th> | <th scope="row" class="dataset-label">Government Jurisdiction</th> |
<td class="dataset-details" property="aglsterms:AglsJuri"> {{ pkg.jurisdiction }}</td> | <td class="dataset-details" property="aglsterms:AglsJuri"> {{ pkg.jurisdiction }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('temporal_coverage') %} | {% if pkg.get('temporal_coverage') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Temporal Coverage</th> | <th scope="row" class="dataset-label">Temporal Coverage</th> |
<td class="dataset-details" property="dct:temporal"> {{ pkg.temporal_coverage }}</td> | <td class="dataset-details" property="dct:temporal"> {{ pkg.temporal_coverage }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('data_state') %} | {% if pkg.get('data_state') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Data State</th> | <th scope="row" class="dataset-label">Data State</th> |
<td class="dataset-details"> {{ pkg.data_state }}</td> | <td class="dataset-details"> {{ pkg.data_state }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% if pkg.get('update_freq') %} | {% if pkg.get('update_freq') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">Update Frequency</th> | <th scope="row" class="dataset-label">Update Frequency</th> |
<td class="dataset-details"> {{ pkg.update_freq }}</td> | <td class="dataset-details"> {{ pkg.update_freq }}</td> |
</tr> | </tr> |
{% endif %} | {% endif %} |
{% set email =h.get_last_active_user(c.pkg_dict['id']).get('email','') %} | {% set email =h.get_last_active_user(c.pkg_dict['id']).get('email','') %} |
<tr> | <tr> |
<th scope="row" class="dataset-label">{{ _('Maintainer') }}</th> | <th scope="row" class="dataset-label">{{ _('Maintainer') }}</th> |
<td class="dataset-details" property="dc:contributor">{{ | <td class="dataset-details" property="dc:contributor">{{ |
h.mail_to(email_address=(email or ' '), name=h.get_last_active_user(c.pkg_dict['id']).get("display_name",'')) }} | h.mail_to(email_address=(email or ' '), name=h.get_last_active_user(c.pkg_dict['id']).get("display_name",'')) }} |
</td> | </td> |
</tr> | </tr> |
<tr> | <tr> |
<th scope="row" class="dataset-label">Publisher/Agency</th> | <th scope="row" class="dataset-label">Publisher/Agency</th> |
<td class="dataset-details" property="dc:publisher">{{ c.pkg_dict['organization']['title']}} | <td class="dataset-details" property="dc:publisher">{{ c.pkg_dict['organization']['title']}} |
</td> | </td> |
</tr> | </tr> |
<tr> | <tr> |
<th scope="row" class="dataset-label">Type</th> | <th scope="row" class="dataset-label">Type</th> |
<td class="dataset-details" property="dc:type">Dataset</td> | <td class="dataset-details" property="dc:type">Dataset</td> |
</tr> | </tr> |
<tr> | <tr> |
<th scope="row" class="dataset-label">Language</th> | <th scope="row" class="dataset-label">Language</th> |
<td class="dataset-details" property="dc:language">English</td> | <td class="dataset-details" property="dc:language">English</td> |
</tr> | </tr> |
</tbody> | </tbody> |
</table> | </table> |
</section> | </section> |
{% endblock %} | {% endblock %} |
{% block secondary_content %} | |
{{ super() }} | |
{% set dataset_extent = h.get_pkg_dict_extra(c.pkg_dict, 'spatial', '') %} | |
{% if dataset_extent %} | |
{% snippet "spatial/snippets/dataset_map_sidebar.html", extent=dataset_extent %} | |
{% endif %} | |
{% endblock %} | |
{% ckan_extends %} | |
{% block secondary_content %} | |
{{ super() }} | |
{% snippet "spatial/snippets/spatial_query.html", default_extent="[[-11, 114], [-42, 154]]" %} | |
{% endblock %} | |