From: CKAN data.gov.au Date: Wed, 25 Sep 2013 09:26:40 +0000 Subject: add varnish config to ignore locales X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=9f64ed1f3b815f474681a119d41cbbaf32da15b8 --- add varnish config to ignore locales --- --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,21 +3,7 @@ - - - - - - - - - - - + --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -5,26 +5,32 @@ - + + + + + - - - - - - - + + + + + + + - - + + - + + + - + @@ -85,28 +91,28 @@ - - + + - + - - + + - + - - + + - + @@ -115,7 +121,52 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -129,6 +180,7 @@ + @@ -139,28 +191,30 @@ - @@ -188,7 +242,7 @@ - + @@ -231,7 +285,7 @@ @@ -248,6 +302,10 @@ - + + + + + + + + + + - - - - + + - - + + @@ -486,6 +567,23 @@ + + @@ -507,24 +605,24 @@ - - + + + + - + - + - + - - @@ -572,7 +670,7 @@ - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + No facets are configured + + + + + + + + + + + + + + + + + + + + + + ckanext-datagovau + + + + + + + + + + + + + + + + + + + + --- /dev/null +++ b/admin/data.gov.au @@ -1,1 +1,96 @@ + + ServerAdmin webmaster@localhost + ServerName data.gov.au + ServerAlias www.data.gov.au, opendata.linkdigital.com.au + + + #allow access to local static content eg. logos + Alias /index.html /var/www/index.html + Alias /logos /var/www/logos + Alias /tools /var/www/tools.shtml + Alias /opendata /var/www/opendata.shtml + Alias /opengov /var/www/opengov.shtml + + DocumentRoot /var/www + + + ExpiresActive On + ExpiresDefault "access plus 1 week" + + + + + Options FollowSymLinks + AllowOverride None + + + + Options -Indexes -FollowSymLinks -MultiViews +Includes + AllowOverride None + + Order allow,deny + allow from all + + AddType text/html .shtml + AddOutputFilter INCLUDES .shtml + + + ErrorLog ${APACHE_LOG_DIR}/error.log + + # Possible values include: debug, info, notice, warn, error, crit, alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/access.log varnishcombined + RewriteEngine On + + RewriteMap lc int:tolower + +# RewriteRule ^/apps/antenna-mate/?$ http://antennamate.com/ [PT] +# RewriteRule ^/apps/postcode-finder/?$ http://www.aus-emaps.com/postcode_finder.php [PT] +# RewriteRule ^/apps/the-australian-cost-of-living-heatmap/?$ http://www.creditcardcompare.com.au/tools/cost-of-living-heatmap/ [PT] + RewriteRule ^/catalogues/?$ /organization [PT] + RewriteRule ^/dataset/“cycling-in-new-south-wales-what-the-data-tells-us”-and-related-data/? /dataset/2809cycling-in-new-south-wales-what-the-data-tells-us2809-and-related-data [PT] + RewriteRule ^/dataset/apvma-pubcris-dataset-for-registered-agricultural-and-veterinary-chemical-products-and-approved-actives/? /dataset/apvma-pubcris-dataset-for-registered-agricultural-and-veterinary-chemical-products-and-approved-acti [PT] + RewriteRule ^/dataset_category/business/?$ /group/business [PT] + RewriteRule ^/dataset_category/community/?$ /group/community [PT] + RewriteRule ^/dataset_category/geography/?$ /group/geography [PT] + RewriteRule ^/dataset_category/government/?$ /group/government [PT] + RewriteRule ^/dataset_category/health/?$ /group/health [PT] + RewriteRule ^/dataset_category/?$ /group [PT] + RewriteRule ^/feed/?$ /feeds/dataset.atom [PT] + RewriteRule ^/jurisdiction/australian-capital-territory/?$ /dataset?q=act [PT] + RewriteRule ^/jurisdiction/new-south-wales/?$ /dataset?q=new+south+wales [PT] + RewriteRule ^/jurisdiction/queensland/?$ /dataset?q=queensland [PT] + RewriteRule ^/jurisdiction/south-australia/?$ /dataset?q=south+australia [PT] + RewriteRule ^/jurisdiction/victoria/?$ /dataset?q=victoria [PT] + RewriteRule ^/jurisdiction/western-australia/?$ /dataset?q=western+australia [PT] + RewriteRule ^/jurisdiction/?$ /dataset [PT] + +# RewriteCond %{QUERY_STRING} ^category=([a-z]+) +# RewriteRule ^/data/?$ /group/${lc:%1}? [PT] + +# RewriteCond %{QUERY_STRING} ^format=Shapefile +# RewriteRule ^/data/?$ /dataset?res_format=shp [PT] + + RewriteRule ^/data/?$ /dataset? [PT] + + RewriteCond %{QUERY_STRING} ^s=(.*) + RewriteRule ^/ /dataset?q=%1 [PT] + + + FilterDeclare OPENDATA + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $text/ + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/javascript + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/json + FilterProvider OPENDATA SUBSTITUTE resp=Content-Type $/xml + FilterChain OPENDATA + Substitute "s#//opendata.linkdigital.com.au#//data.gov.au#" + + WSGIScriptAlias / /var/lib/ckan/dga/pyenv/bin/dga.py + WSGIPassAuthorization On + + ErrorLog /var/log/apache2/ckandga.error.log + CustomLog /var/log/apache2/ckandga.custom.log combined + + --- /dev/null +++ b/admin/default.vcl @@ -1,1 +1,175 @@ +# This is a basic VCL configuration file for varnish. See the vcl(7) +# man page for details on VCL syntax and semantics. +# +# Default backend definition. Set this to point to your content +# server. +# +backend default { + .host = "127.0.0.1"; + .port = "8080"; +} +sub vcl_fetch { + set beresp.grace = 1h; + + if (beresp.http.content-type ~ "(text|application)") { + set beresp.do_gzip = true; + } + if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { + unset beresp.http.set-cookie; + } +} +sub vcl_recv { + if (req.url ~ "^/_tracking") { + return (pass); + } + if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { + return(lookup); + } + if (req.http.Cookie) { + set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing + + if (req.http.Cookie == "") { + remove req.http.Cookie; + } + } +} +sub vcl_hash { + # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl + set req.url = regsub(req.url, "/../|/.._../", "/"); + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (hash); +} +sub vcl_deliver { + if (!resp.http.Vary) { + set resp.http.Vary = "Accept-Encoding"; + } else if (resp.http.Vary !~ "(?i)Accept-Encoding") { + set resp.http.Vary = resp.http.Vary + ",Accept-Encoding"; + } + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } +} + +# +# Below is a commented-out copy of the default VCL logic. If you +# redefine any of these subroutines, the built-in logic will be +# appended to your code. +# sub vcl_recv { +# if (req.restarts == 0) { +# if (req.http.x-forwarded-for) { +# set req.http.X-Forwarded-For = +# req.http.X-Forwarded-For + ", " + client.ip; +# } else { +# set req.http.X-Forwarded-For = client.ip; +# } +# } +# if (req.request != "GET" && +# req.request != "HEAD" && +# req.request != "PUT" && +# req.request != "POST" && +# req.request != "TRACE" && +# req.request != "OPTIONS" && +# req.request != "DELETE") { +# /* Non-RFC2616 or CONNECT which is weird. */ +# return (pipe); +# } +# if (req.request != "GET" && req.request != "HEAD") { +# /* We only deal with GET and HEAD by default */ +# return (pass); +# } +# if (req.http.Authorization || req.http.Cookie) { +# /* Not cacheable by default */ +# return (pass); +# } +# return (lookup); +# } +# +# sub vcl_pipe { +# # Note that only the first request to the backend will have +# # X-Forwarded-For set. If you use X-Forwarded-For and want to +# # have it set for all requests, make sure to have: +# # set bereq.http.connection = "close"; +# # here. It is not set by default as it might break some broken web +# # applications, like IIS with NTLM authentication. +# return (pipe); +# } +# +# sub vcl_pass { +# return (pass); +# } +# +# sub vcl_hash { +# hash_data(req.url); +# if (req.http.host) { +# hash_data(req.http.host); +# } else { +# hash_data(server.ip); +# } +# return (hash); +# } +# +# sub vcl_hit { +# return (deliver); +# } +# +# sub vcl_miss { +# return (fetch); +# } +# +# sub vcl_fetch { +# if (beresp.ttl <= 0s || +# beresp.http.Set-Cookie || +# beresp.http.Vary == "*") { +# /* +# * Mark as "Hit-For-Pass" for the next 2 minutes +# */ +# set beresp.ttl = 120 s; +# return (hit_for_pass); +# } +# return (deliver); +# } +# +# sub vcl_deliver { +# return (deliver); +# } +# +# sub vcl_error { +# set obj.http.Content-Type = "text/html; charset=utf-8"; +# set obj.http.Retry-After = "5"; +# synthetic {" +# +# +# +# +# "} + obj.status + " " + obj.response + {" +# +# +#

Error "} + obj.status + " " + obj.response + {"

+#

"} + obj.response + {"

+#

Guru Meditation:

+#

XID: "} + req.xid + {"

+#
+#

Varnish cache server

+# +# +# "}; +# return (deliver); +# } +# +# sub vcl_init { +# return (ok); +# } +# +# sub vcl_fini { +# return (ok); +# } + --- /dev/null +++ b/admin/deflate.conf @@ -1,1 +1,12 @@ + + AddOutputFilterByType DEFLATE text/html application/xhtml+xml text/plain text/xml text/css application/json text/javascript application/javascript application/x-javascript + BrowserMatch ^Mozilla/4 gzip-only-text/html + BrowserMatch ^Mozilla/4\.0[678] no-gzip + BrowserMatch \bMSIE !no-gzip !gzip-only-text/htm + # Don't compress images + SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png)$ no-gzip dont-vary + + # DeflateCompressionLevel 9 + + --- a/ckanext/datagovau/plugin.py +++ b/ckanext/datagovau/plugin.py @@ -1,24 +1,32 @@ import logging import ckan.plugins as plugins +import ckan.lib as lib +import ckan.lib.dictization.model_dictize as model_dictize import ckan.plugins.toolkit as tk +import ckan.model as model +from pylons import config +#parse the activity feed for last active non-system user +def get_last_active_user(id): + system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')}) + user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']] + user = None + if len(user_list) > 0: + user = user_list[0].get('user_id', None) + if user is None: + return system_user + else: + return lib.helpers.get_action('user_show',{'id':user}) -# get_action will be deprecated soon, this is the replacement -def get_user(user): - user_name = unicode(user) - if user_name: - user_obj = model.User.get(id) - context['user_obj'] = user_obj - if user_obj is None: - raise NotFound - else: - raise NotFound +# get user created datasets and those they have edited +def get_user_datasets(user_dict): + created_datasets_list = user_dict['datasets'] + active_datasets_list = [x['data']['package'] for x in + lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] + return created_datasets_list + active_datasets_list - return model_dictize.user_dictize(user_obj,context) - - -class ExampleIDatasetFormPlugin(plugins.SingletonPlugin, +class DataGovAuPlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm): '''An example IDatasetForm CKAN plugin. @@ -28,18 +36,6 @@ plugins.implements(plugins.IConfigurer, inherit=False) plugins.implements(plugins.IDatasetForm, inherit=False) plugins.implements(plugins.ITemplateHelpers, inherit=False) - - # These record how many times methods that this plugin's methods are - # called, for testing purposes. - num_times_new_template_called = 0 - num_times_read_template_called = 0 - num_times_edit_template_called = 0 - num_times_search_template_called = 0 - num_times_history_template_called = 0 - num_times_package_form_called = 0 - num_times_check_data_dict_called = 0 - num_times_setup_template_variables_called = 0 - def update_config(self, config): # Add this plugin's templates dir to CKAN's extra_template_paths, so @@ -53,7 +49,7 @@ # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' def get_helpers(self): - return {'getuser': get_user} + return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets} def is_fallback(self): # Return True to register this plugin as the default handler for @@ -67,17 +63,17 @@ def create_package_schema(self): - schema = super(ExampleIDatasetFormPlugin, self).create_package_schema() + schema = super(DataGovAuPlugin, self).create_package_schema() schema = self._modify_package_schema(schema) return schema def update_package_schema(self): - schema = super(ExampleIDatasetFormPlugin, self).update_package_schema() + schema = super(DataGovAuPlugin, self).update_package_schema() schema = self._modify_package_schema(schema) return schema def show_package_schema(self): - schema = super(ExampleIDatasetFormPlugin, self).show_package_schema() + schema = super(DataGovAuPlugin, self).show_package_schema() # Don't show vocab tags mixed in with normal 'free' tags # (e.g. on dataset pages, or on the search page) @@ -140,25 +136,25 @@ # called. def setup_template_variables(self, context, data_dict): - return super(ExampleIDatasetFormPlugin, self).setup_template_variables( + return super(DataGovAuPlugin, self).setup_template_variables( context, data_dict) def new_template(self): - return super(ExampleIDatasetFormPlugin, self).new_template() + return super(DataGovAuPlugin, self).new_template() def read_template(self): - return super(ExampleIDatasetFormPlugin, self).read_template() + return super(DataGovAuPlugin, self).read_template() def edit_template(self): - return super(ExampleIDatasetFormPlugin, self).edit_template() + return super(DataGovAuPlugin, self).edit_template() def search_template(self): - return super(ExampleIDatasetFormPlugin, self).search_template() + return super(DataGovAuPlugin, self).search_template() def history_template(self): - return super(ExampleIDatasetFormPlugin, self).history_template() + return super(DataGovAuPlugin, self).history_template() def package_form(self): - return super(ExampleIDatasetFormPlugin, self).package_form() + return super(DataGovAuPlugin, self).package_form() --- a/ckanext/datagovau/templates/package/read.html +++ b/ckanext/datagovau/templates/package/read.html @@ -1,13 +1,19 @@ {% ckan_extends %} +{% block primary_content_inner %} +{{ super() }} +
+ {{ h.disqus_comments() }} +
+{% endblock %} {% block package_additional_info %} -
+

{{ _('Additional Info') }}

@@ -38,17 +44,11 @@ {% endif %} - {% if pkg.get('Agency Program') %} - - - - - {% endif %} - {% if pkg.get('DCTERMS.Source.URI') %} + {% if pkg.get('url') %} - {% endif %} @@ -67,13 +67,6 @@ {% endif %} - {% if pkg.get('Geospatial Coverage') %} - - - - - - {% endif %} {% if pkg.get('granularity') %} @@ -81,43 +74,19 @@ {% endif %} - {% if pkg.get('Granularity') %} - - - - - - {% endif %} {% if pkg.get('jurisdiction') %} - {% endif %} - {% if pkg.get('AGLSTERMS.Jurisdiction') %} - - - - - - {% endif %} {% if pkg.get('temporal_coverage') %} - {% endif %} - {% if pkg.get('Temporal Coverage') %} - - - - - - {% endif %} {% if pkg.get('data_state') %} - @@ -130,26 +99,27 @@ {% endif %} - {% if pkg.get('Update Frequency') %} - - - - - {% endif %} - {% if pkg.maintainer_email %} +{% set email =h.get_last_active_user(c.pkg_dict['id']).get('email','') %} - {% elif pkg.maintainer %} - - + + - {% endif %} + + + + + + + +
{{ pkg.agency_program }}
Agency Program {{ pkg.get('Agency Program') }}
{{ _('Source') }}{{ h.link_to(pkg.get('DCTERMS.Source.URI'), - pkg.get('DCTERMS.Source.URI'), rel='dct:source', target='_blank') }} + {{ h.link_to(pkg.get('url'), + pkg.get('url'), rel='dct:source', target='_blank') }}
{{ pkg.spatial_coverage }}
Geospatial Coverage {{ pkg.get('Geospatial Coverage') }}
{{ pkg.granularity }}
Data Granularity {{ pkg.get('Granularity') }}
Government Jurisdiction {{ pkg.jurisdiction }}
Government Jurisdiction {{ pkg.get('AGLSTERMS.Jurisdiction') }}
Temporal Coverage {{ pkg.temporal_coverage }}
Temporal Coverage {{ pkg.get('Temporal Coverage') }}
Data State {{ pkg.data_state }} {{ pkg.update_freq }}
Update Frequency {{ pkg.get('Update Frequency') }}
{{ _('Maintainer') }} {{ - h.mail_to(email_address=pkg.maintainer_email, name=pkg.maintainer) }} + h.mail_to(email_address=(email or ' '), name=h.get_last_active_user(c.pkg_dict['id']).get("display_name",'')) }}
{{ _('Maintainer') }}{{ pkg.maintainer }}Publisher/Agency{{ c.pkg_dict['organization']['title']}} +
TypeDataset
LanguageEnglish
--- a/ckanext/datagovau/templates/package/read.rdf +++ b/ckanext/datagovau/templates/package/read.rdf @@ -24,7 +24,17 @@ ${c.pkg_dict['metadata_created']} ${c.pkg_dict['metadata_modified']} en + + + ${c.pkg_dict['license_url']} + + + + ${c.pkg_dict['license_id']} + + + ${ tag_dict["name"] } @@ -69,51 +79,30 @@ ${ c.pkg_dict['organization']['title'] } - + - ${h.get_action("user_show",{"id":username})["display_name"]} - + ${h.get_last_active_user(c.pkg_dict['id'])["display_name"]} + - - ${c.pkg_dict.get('DCTERMS.Source.URI') } - - + ${c.pkg_dict.get('url')} ${c.pkg_dict.contact_point } ${ c.pkg_dict.spatial_coverage } - ${ c.pkg_dict.get('Geospatial Coverage') } ${ c.pkg_dict.jurisdiction } - ${ c.pkg_dict.get('AGLSTERMS.Jurisdiction') } - ${ c.pkg_dict.temporal_coverage } - ${ c.pkg_dict.get('Temporal Coverage') } + ${ c.pkg_dict.get('temporal_coverage') } Data State ${ c.pkg_dict.get('data_state') } - Update Frequency ${ c.pkg_dict.get('update_freq') } - - - - - Update Frequency - ${ c.pkg_dict.get('Update Frequency') } - - - - - - Agency Program - ${ c.pkg_dict.get('Agency Program') } @@ -122,20 +111,12 @@ ${ c.pkg_dict.get('agency_program') } - - - - Data Granularity - ${ c.pkg_dict.get('Granularity') } - - Data Granularity ${ c.pkg_dict.get('granularity') } - --- a/ckanext/datagovau/templates/package/read_base.html +++ b/ckanext/datagovau/templates/package/read_base.html @@ -14,11 +14,11 @@ -{% if pkg.get('DCTERMS.Source.URI') %}{% endif %} + - - - + + + @@ -30,7 +30,7 @@ {% block package_info %}
-

{{ pkg.title or pkg.name }}

+

{{ pkg.title or pkg.name }}

{{ _('Followers') }}
--- a/ckanext/datagovau/templates/package/resource_read.html +++ b/ckanext/datagovau/templates/package/resource_read.html @@ -14,11 +14,11 @@ -{% if pkg.get('DCTERMS.Source.URI') %}{% endif %} + - - - + + + @@ -26,7 +26,6 @@ - --- a/ckanext/datagovau/templates/package/snippets/package_basic_fields.html +++ b/ckanext/datagovau/templates/package/snippets/package_basic_fields.html @@ -3,6 +3,7 @@ {% block package_basic_fields_org %} {{ super() }} + {{ form.input('agency_program', label=_('Agency Program'), id='field-agency_program', placeholder=_('Name of the Agency Program that generated the data if relevant'), --- a/ckanext/datagovau/templates/package/snippets/package_metadata_fields.html +++ b/ckanext/datagovau/templates/package/snippets/package_metadata_fields.html @@ -10,8 +10,9 @@ {# https://github.com/okfn/ckan/blob/master/ckan/templates/macros/form.html documents the macros for fields #} {% block package_metadata_fields %} - - +{{ form.input('url', label=_('Source URL'), id='field-url', +placeholder=_('URL where dataset came from or more information can be obtained'), +value=data.url, error=errors.url, classes=['control-medium']) }} {{ super() }} {% endblock %} --- /dev/null +++ b/ckanext/datagovau/templates/package/snippets/resource_item.html @@ -1,1 +1,54 @@ +{% ckan_extends %} +{% set url = h.url_for(controller='package', action='resource_read', id=pkg.name, resource_id=res.id) %} +
  • + {% block resource_item_title %} + + {{ h.resource_display_name(res) | truncate(50) }}{{ res.format }} + {{ h.popular('views', res.tracking_summary.total, min=10) }} + + {% endblock %} +

    + {% if res.description %} + {{ h.markdown_extract(res.description, extract_length=80) }} + {% else %} + {{ _('No description for this resource') }} + {% endif %} +

    + {% block resource_item_explore %} +
    + + + {{ _('Explore') }} + + + +
    + {% endblock %} +
    • --- /dev/null +++ b/ckanext/datagovau/templates/user/dashboard_datasets.html @@ -1,1 +1,16 @@ +{% ckan_extends %} +{% block primary_content_inner %} +

    {{ _('My Datasets') }}

    + {% if h.get_user_datasets(c.user_dict) %} + {% snippet 'snippets/package_list.html', packages=h.get_user_datasets(c.user_dict) %} + {% else %} +

    + You haven\'t created/edited any datasets. + {% if h.check_access('package_create') %} + {% link_for _('Create one now?'), controller='package', action='new' %} + {% endif %} +

    + {% endif %} +{% endblock %} + --- /dev/null +++ b/ckanext/datagovau/templates/user/read.html @@ -1,1 +1,29 @@ +{% ckan_extends %} + +{% block primary_content_inner %} +

    + {% block page_heading %}{{ _('Datasets') }}{% endblock %} +

    + {% block package_list %} + {% if h.get_user_datasets(c.user_dict) %} + {% snippet 'snippets/package_list.html', packages=h.get_user_datasets(c.user_dict) %} + {% else %} + + {% if c.is_myself %} +

    + {{ _('You haven\'t created any datasets.') }} + {% if h.check_access('package_create') %} + {% link_for _('Create one now?'), controller='package', action='new' %} + {% endif %} +

    + {% else %} +

    + {{ _('User hasn\'t created any datasets.') }} +

    + {% endif %} + {% endif %} + {% endblock %} +{% endblock %} + + --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ entry_points=\ """ [ckan.plugins] - datagovau=ckanext.datagovau.plugin:ExampleIDatasetFormPlugin + datagovau=ckanext.datagovau.plugin:DataGovAuPlugin """, )