From: Alex Sadleir Date: Wed, 09 Oct 2013 11:40:58 +0000 Subject: statistics helper functions X-Git-Url: http://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=937ca9b386a01e5daa43696ec57cff39f99d2791 --- statistics helper functions --- --- /dev/null +++ b/.idea/.name @@ -1,1 +1,1 @@ - +ckanext-datagovau --- /dev/null +++ b/.idea/compiler.xml @@ -1,1 +1,24 @@ + + + + + + --- /dev/null +++ b/.idea/encodings.xml @@ -1,1 +1,6 @@ + + + + + --- /dev/null +++ b/.idea/misc.xml @@ -1,1 +1,9 @@ + + + + + + + + --- /dev/null +++ b/.idea/modules.xml @@ -1,1 +1,10 @@ + + + + + + + + + --- /dev/null +++ b/.idea/other.xml @@ -1,1 +1,8 @@ + + + + + + --- /dev/null +++ b/.idea/scopes/scope_settings.xml @@ -1,1 +1,5 @@ - + + + + --- /dev/null +++ b/.idea/testrunner.xml @@ -1,1 +1,9 @@ + + + + + + --- /dev/null +++ b/.idea/vcs.xml @@ -1,1 +1,8 @@ + + + + + + + --- /dev/null +++ b/.idea/workspace.xml @@ -1,1 +1,881 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + localhost + 5050 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No facets are configured + + + + + + + + + + + + + + + + + + + + + + ckanext-datagovau + + + + + + + + + + + + + + + + + + + + + + + --- a/README.rst +++ b/README.rst @@ -1,66 +1,27 @@ -This CKAN Extension demonstrates some common patterns for customising a CKAN instance. +This CKAN Extension customises a CKAN instance for the hosting of data.gov.au. It comprises: -* A CKAN Extension "plugin" at ``ckanext/example/plugin.py`` which, when +* A CKAN Extension "plugin" at ``ckanext/datagovau/plugin.py`` which, when loaded, overrides various settings in the core ``ini``-file to provide: + * A path to local customisations of the core templates + * A custom Package edit form that defaults to cc-by licence + * A custom n3/rdf output format - * A path to local customisations of the core templates and stylesheets - * A "stream filter" that replaces arbitrary strings in rendered templates - * A "route" to override and extend the default behaviour of a core CKAN page - -* A custom Pylons controller for overriding some core CKAN behaviour - -* A custom Package edit form - -* A custom Group edit form - -* A plugin that allows for custom forms to be used for datasets based on - their "type". - -* A custom User registration and edition form - -* Some simple template customisations +* A cut down licenses.json file Installation ============ To install this package, from your CKAN virtualenv, run the following from your CKAN base folder (e.g. ``pyenv/``):: - pip install -e git+https://github.com/okfn/ckanext-example#egg=ckanext-example + pip install -e git+https://github.com/okfn/ckanext-datagovau#egg=ckanext-datagovau -Then activate it by setting ``ckan.plugins = example`` in your main ``ini``-file. +Then activate it by setting ``ckan.plugins = datagovau`` in your main ``ini``-file. + +To add the cut down licenses.json set ``licenses_group_url = http://%(ckan.site_url)/licenses.json`` +or copy ``ckanext/datagovau/theme/public/licenses.json`` to the same folder as your CKAN config ini file +and set ``licenses_group_url = file://%(here)s/licenses.json`` -Orientation -=========== -* Examine the source code, starting with ``ckanext/example/plugin.py`` - -* To understand the nuts and bolts of this file, which is a CKAN - *Extension*, read in conjunction with the "Extension - documentation": http://docs.ckan.org/en/latest/plugins.html - -* One thing the extension does is set the values of - ``extra_public_paths`` and ``extra_template_paths`` in the CKAN - config, which are "documented - here": http://docs.ckan.org/en/latest/configuration.html#extra-template-paths - -* These are set to point at directories within - ``ckanext/example/theme/`` (in this package). Here we: - * override the home page HTML ``ckanext/example/theme/templates/home/index.html`` - * provide some extra style by serving ``extra.css`` (which is loaded using the ``ckan.template_head_end`` option - * customise the navigation and header of the main template in the file ``layout.html``. - - The latter file is a great place to make global theme alterations. - It uses the _layout template_ pattern "described in the Genshi - documentation":http://genshi.edgewall.org/wiki/GenshiTutorial#AddingaLayoutTemplate. - This allows you to use Xpath selectors to override snippets of HTML - globally. - -* The custom package edit form at ``package_form.py`` follows a deprecated - way to make a form (using FormAlchemy). This part of the Example Theme needs - updating. In the meantime, follow the instructions at: - http://readthedocs.org/docs/ckan/en/latest/forms.html - - --- /dev/null +++ b/admin/data.gov.au @@ -1,1 +1,101 @@ + + ServerAdmin webmaster@localhost + ServerName data.gov.au + ServerAlias www.data.gov.au, opendata.linkdigital.com.au + + + #allow access to local static content eg. logos + Alias /index.html /var/www/index.html + Alias /logos /var/www/logos + Alias /tools /var/www/tools.shtml + Alias /opendata /var/www/opendata.shtml + Alias /opengov /var/www/opengov.shtml + + DocumentRoot /var/www + + + ExpiresActive On + ExpiresDefault "access plus 1 week" + + +AddType application/octet-stream .woff + + SetEnvIf Origin "^http(s)?://(.+\.)?(data.gov.au)$" origin_is=$0 + Header set Access-Control-Allow-Origin %{origin_is}e env=origin_is + + + + Options FollowSymLinks + AllowOverride None + + + + Options -Indexes -FollowSymLinks -MultiViews +Includes + AllowOverride None + + Order allow,deny + allow from all + + AddType text/html .shtml + AddOutputFilter INCLUDES .shtml + + + ErrorLog ${APACHE_LOG_DIR}/error.log + + # Possible values include: debug, info, notice, warn, error, crit, alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/access.log varnishcombined + RewriteEngine On + + RewriteMap lc int:tolower + +# RewriteRule ^/apps/antenna-mate/?$ http://antennamate.com/ [PT] +# RewriteRule ^/apps/postcode-finder/?$ http://www.aus-emaps.com/postcode_finder.php [PT] +# RewriteRule ^/apps/the-australian-cost-of-living-heatmap/?$ http://www.creditcardcompare.com.au/tools/cost-of-living-heatmap/ [PT] + RewriteRule ^/catalogues/?$ /organization [PT] + RewriteRule ^/dataset/“cycling-in-new-south-wales-what-the-data-tells-us”-and-related-data/? /dataset/2809cycling-in-new-south-wales-what-the-data-tells-us2809-and-related-data [PT] + RewriteRule ^/dataset/apvma-pubcris-dataset-for-registered-agricultural-and-veterinary-chemical-products-and-approved-actives/? /dataset/apvma-pubcris-dataset-for-registered-agricultural-and-veterinary-chemical-products-and-approved-acti [PT] + RewriteRule ^/dataset_category/business/?$ /group/business [PT] + RewriteRule ^/dataset_category/community/?$ /group/community [PT] + RewriteRule ^/dataset_category/geography/?$ /group/geography [PT] + RewriteRule ^/dataset_category/government/?$ /group/government [PT] + RewriteRule ^/dataset_category/health/?$ /group/health [PT] + RewriteRule ^/dataset_category/?$ /group [PT] + RewriteRule ^/feed/?$ /feeds/dataset.atom [PT] + RewriteRule ^/jurisdiction/australian-capital-territory/?$ /dataset?q=act [PT] + RewriteRule ^/jurisdiction/new-south-wales/?$ /dataset?q=new+south+wales [PT] + RewriteRule ^/jurisdiction/queensland/?$ /dataset?q=queensland [PT] + RewriteRule ^/jurisdiction/south-australia/?$ /dataset?q=south+australia [PT] + RewriteRule ^/jurisdiction/victoria/?$ /dataset?q=victoria [PT] + RewriteRule ^/jurisdiction/western-australia/?$ /dataset?q=western+australia [PT] + RewriteRule ^/jurisdiction/?$ /dataset [PT] + +# RewriteCond %{QUERY_STRING} ^category=([a-z]+) +# RewriteRule ^/data/?$ /group/${lc:%1}? [PT] + +# RewriteCond %{QUERY_STRING} ^format=Shapefile +# RewriteRule ^/data/?$ /dataset?res_format=shp [PT] + + RewriteRule ^/data/?$ /dataset? [PT] + + RewriteCond %{QUERY_STRING} ^s=(.*) + RewriteRule ^/ /dataset?q=%1 [PT] + + + FilterDeclare OPENDATA + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $text/ + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/javascript + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/json + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/xml + FilterChain OPENDATA + Substitute "s#//opendata.linkdigital.com.au#//data.gov.au#" + + WSGIScriptAlias / /var/lib/ckan/dga/pyenv/bin/dga.py + WSGIPassAuthorization On + + ErrorLog /var/log/apache2/ckandga.error.log + CustomLog /var/log/apache2/ckandga.custom.log combined + + --- /dev/null +++ b/admin/default.vcl @@ -1,1 +1,178 @@ +# This is a basic VCL configuration file for varnish. See the vcl(7) +# man page for details on VCL syntax and semantics. +# +# Default backend definition. Set this to point to your content +# server. +# +backend default { + .host = "127.0.0.1"; + .port = "8080"; +} +sub vcl_fetch { + set beresp.grace = 1h; + + if (beresp.http.content-type ~ "(text|application)") { + set beresp.do_gzip = true; + } + if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { + unset beresp.http.set-cookie; + } +} +sub vcl_recv { + if (req.url ~ "^/_tracking") { + return (pass); + } + if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { + return(lookup); + } +if (req.url ~ "/(..|.._..)/") { + set req.url = regsub(req.url, "/((?!js)..|.._..)/", "/"); +} + if (req.http.Cookie) { + set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing + + if (req.http.Cookie == "") { + remove req.http.Cookie; + } + } +} +sub vcl_hash { + # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl + set req.url = regsub(req.url, "(?:(.com|.au))/((?!js)..|.._..)/", "/"); + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (hash); +} +sub vcl_deliver { + if (!resp.http.Vary) { + set resp.http.Vary = "Accept-Encoding"; + } else if (resp.http.Vary !~ "(?i)Accept-Encoding") { + set resp.http.Vary = resp.http.Vary + ",Accept-Encoding"; + } + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } +} + +# +# Below is a commented-out copy of the default VCL logic. If you +# redefine any of these subroutines, the built-in logic will be +# appended to your code. +# sub vcl_recv { +# if (req.restarts == 0) { +# if (req.http.x-forwarded-for) { +# set req.http.X-Forwarded-For = +# req.http.X-Forwarded-For + ", " + client.ip; +# } else { +# set req.http.X-Forwarded-For = client.ip; +# } +# } +# if (req.request != "GET" && +# req.request != "HEAD" && +# req.request != "PUT" && +# req.request != "POST" && +# req.request != "TRACE" && +# req.request != "OPTIONS" && +# req.request != "DELETE") { +# /* Non-RFC2616 or CONNECT which is weird. */ +# return (pipe); +# } +# if (req.request != "GET" && req.request != "HEAD") { +# /* We only deal with GET and HEAD by default */ +# return (pass); +# } +# if (req.http.Authorization || req.http.Cookie) { +# /* Not cacheable by default */ +# return (pass); +# } +# return (lookup); +# } +# +# sub vcl_pipe { +# # Note that only the first request to the backend will have +# # X-Forwarded-For set. If you use X-Forwarded-For and want to +# # have it set for all requests, make sure to have: +# # set bereq.http.connection = "close"; +# # here. It is not set by default as it might break some broken web +# # applications, like IIS with NTLM authentication. +# return (pipe); +# } +# +# sub vcl_pass { +# return (pass); +# } +# +# sub vcl_hash { +# hash_data(req.url); +# if (req.http.host) { +# hash_data(req.http.host); +# } else { +# hash_data(server.ip); +# } +# return (hash); +# } +# +# sub vcl_hit { +# return (deliver); +# } +# +# sub vcl_miss { +# return (fetch); +# } +# +# sub vcl_fetch { +# if (beresp.ttl <= 0s || +# beresp.http.Set-Cookie || +# beresp.http.Vary == "*") { +# /* +# * Mark as "Hit-For-Pass" for the next 2 minutes +# */ +# set beresp.ttl = 120 s; +# return (hit_for_pass); +# } +# return (deliver); +# } +# +# sub vcl_deliver { +# return (deliver); +# } +# +# sub vcl_error { +# set obj.http.Content-Type = "text/html; charset=utf-8"; +# set obj.http.Retry-After = "5"; +# synthetic {" +# +# +# +# +# "} + obj.status + " " + obj.response + {" +# +# +#

Error "} + obj.status + " " + obj.response + {"

"} + obj.response + {"

Guru Meditation:

XID: "} + req.xid + {"

Varnish cache server

+# +# +# "}; +# return (deliver); +# } +# +# sub vcl_init { +# return (ok); +# } +# +# sub vcl_fini { +# return (ok); +# } + --- /dev/null +++ b/admin/deflate.conf @@ -1,1 +1,12 @@ + + AddOutputFilterByType DEFLATE text/html application/xhtml+xml text/plain text/xml text/css application/json text/javascript application/javascript application/x-javascript + BrowserMatch ^Mozilla/4 gzip-only-text/html + BrowserMatch ^Mozilla/4\.0[678] no-gzip + BrowserMatch \bMSIE !no-gzip !gzip-only-text/htm + # Don't compress images + SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png)$ no-gzip dont-vary + + # DeflateCompressionLevel 9 + + --- /dev/null +++ b/admin/development.ini @@ -1,1 +1,365 @@ - +# +# ckan - Pylons configuration +# +# The %(here)s variable will be replaced with the parent directory of this file +# +[DEFAULT] + +# Change debug to true when doing CKAN development, it enables Pylons' +# interactive debugging tool, makes Fanstatic serve unminified JS and CSS +# files, and enables CKAN templates' debugging features. +# +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* +# Debug mode will enable the interactive debugging tool, allowing ANYONE to +# execute malicious code after an exception is raised. +debug = False + +email_to = ddg@alert.links.com.au +error_email_from = data.gov@finance.gov.au + +# The SMTP server to connect to when sending emails to users, with optional +# port (default: 25). For example: lavabit.com, or smtp.gmail.com:587. To +# connect to a local sendmail process enter 'localhost'. +#smtp.server = localhost +