add varnish config to ignore locales
add varnish config to ignore locales

# This is a basic VCL configuration file for varnish. See the vcl(7) # This is a basic VCL configuration file for varnish. See the vcl(7)
# man page for details on VCL syntax and semantics. # man page for details on VCL syntax and semantics.
# #
# Default backend definition. Set this to point to your content # Default backend definition. Set this to point to your content
# server. # server.
# #
backend default { backend default {
.host = "127.0.0.1"; .host = "127.0.0.1";
.port = "8080"; .port = "8080";
} }
   
sub vcl_fetch { sub vcl_fetch {
set beresp.grace = 1h; set beresp.grace = 1h;
if (beresp.http.content-type ~ "(text|application)") { if (beresp.http.content-type ~ "(text|application)") {
set beresp.do_gzip = true; set beresp.do_gzip = true;
} }
if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") {
unset beresp.http.set-cookie; unset beresp.http.set-cookie;
} }
} }
sub vcl_recv { sub vcl_recv {
if (req.url ~ "^/_tracking") { if (req.url ~ "^/_tracking") {
return (pass); return (pass);
} }
if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") {
return(lookup); return(lookup);
} }
if (req.http.Cookie) { if (req.http.Cookie) {
set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing
   
if (req.http.Cookie == "") { if (req.http.Cookie == "") {
remove req.http.Cookie; remove req.http.Cookie;
} }
} }
} }
  sub vcl_hash {
  # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl
  set req.url = regsub(req.url, "/../|/.._../", "/");
  hash_data(req.url);
  if (req.http.host) {
  hash_data(req.http.host);
  } else {
  hash_data(server.ip);
  }
  return (hash);
  }
sub vcl_deliver { sub vcl_deliver {
if (!resp.http.Vary) { if (!resp.http.Vary) {
set resp.http.Vary = "Accept-Encoding"; set resp.http.Vary = "Accept-Encoding";
} else if (resp.http.Vary !~ "(?i)Accept-Encoding") { } else if (resp.http.Vary !~ "(?i)Accept-Encoding") {
set resp.http.Vary = resp.http.Vary + ",Accept-Encoding"; set resp.http.Vary = resp.http.Vary + ",Accept-Encoding";
} }
if (obj.hits > 0) { if (obj.hits > 0) {
set resp.http.X-Cache = "HIT"; set resp.http.X-Cache = "HIT";
} else { } else {
set resp.http.X-Cache = "MISS"; set resp.http.X-Cache = "MISS";
} }
} }
   
# #
# Below is a commented-out copy of the default VCL logic. If you # Below is a commented-out copy of the default VCL logic. If you
# redefine any of these subroutines, the built-in logic will be # redefine any of these subroutines, the built-in logic will be
# appended to your code. # appended to your code.
# sub vcl_recv { # sub vcl_recv {
# if (req.restarts == 0) { # if (req.restarts == 0) {
# if (req.http.x-forwarded-for) { # if (req.http.x-forwarded-for) {
# set req.http.X-Forwarded-For = # set req.http.X-Forwarded-For =
# req.http.X-Forwarded-For + ", " + client.ip; # req.http.X-Forwarded-For + ", " + client.ip;
# } else { # } else {
# set req.http.X-Forwarded-For = client.ip; # set req.http.X-Forwarded-For = client.ip;
# } # }
# } # }
# if (req.request != "GET" && # if (req.request != "GET" &&
# req.request != "HEAD" && # req.request != "HEAD" &&
# req.request != "PUT" && # req.request != "PUT" &&
# req.request != "POST" && # req.request != "POST" &&
# req.request != "TRACE" && # req.request != "TRACE" &&
# req.request != "OPTIONS" && # req.request != "OPTIONS" &&
# req.request != "DELETE") { # req.request != "DELETE") {
# /* Non-RFC2616 or CONNECT which is weird. */ # /* Non-RFC2616 or CONNECT which is weird. */
# return (pipe); # return (pipe);
# } # }
# if (req.request != "GET" && req.request != "HEAD") { # if (req.request != "GET" && req.request != "HEAD") {
# /* We only deal with GET and HEAD by default */ # /* We only deal with GET and HEAD by default */
# return (pass); # return (pass);
# } # }
# if (req.http.Authorization || req.http.Cookie) { # if (req.http.Authorization || req.http.Cookie) {
# /* Not cacheable by default */ # /* Not cacheable by default */
# return (pass); # return (pass);
# } # }
# return (lookup); # return (lookup);
# } # }
# #
# sub vcl_pipe { # sub vcl_pipe {
# # Note that only the first request to the backend will have # # Note that only the first request to the backend will have
# # X-Forwarded-For set. If you use X-Forwarded-For and want to # # X-Forwarded-For set. If you use X-Forwarded-For and want to
# # have it set for all requests, make sure to have: # # have it set for all requests, make sure to have:
# # set bereq.http.connection = "close"; # # set bereq.http.connection = "close";
# # here. It is not set by default as it might break some broken web # # here. It is not set by default as it might break some broken web
# # applications, like IIS with NTLM authentication. # # applications, like IIS with NTLM authentication.
# return (pipe); # return (pipe);
# } # }
# #
# sub vcl_pass { # sub vcl_pass {
# return (pass); # return (pass);
# } # }
# #
# sub vcl_hash { # sub vcl_hash {
# hash_data(req.url); # hash_data(req.url);
# if (req.http.host) { # if (req.http.host) {
# hash_data(req.http.host); # hash_data(req.http.host);
# } else { # } else {
# hash_data(server.ip); # hash_data(server.ip);
# } # }
# return (hash); # return (hash);
# } # }
# #
# sub vcl_hit { # sub vcl_hit {
# return (deliver); # return (deliver);
# } # }
# #
# sub vcl_miss { # sub vcl_miss {
# return (fetch); # return (fetch);
# } # }
# #
# sub vcl_fetch { # sub vcl_fetch {
# if (beresp.ttl <= 0s || # if (beresp.ttl <= 0s ||
# beresp.http.Set-Cookie || # beresp.http.Set-Cookie ||
# beresp.http.Vary == "*") { # beresp.http.Vary == "*") {
# /* # /*
# * Mark as "Hit-For-Pass" for the next 2 minutes # * Mark as "Hit-For-Pass" for the next 2 minutes
# */ # */
# set beresp.ttl = 120 s; # set beresp.ttl = 120 s;
# return (hit_for_pass); # return (hit_for_pass);
# } # }
# return (deliver); # return (deliver);
# } # }
# #
# sub vcl_deliver { # sub vcl_deliver {
# return (deliver); # return (deliver);
# } # }
# #
# sub vcl_error { # sub vcl_error {
# set obj.http.Content-Type = "text/html; charset=utf-8"; # set obj.http.Content-Type = "text/html; charset=utf-8";
# set obj.http.Retry-After = "5"; # set obj.http.Retry-After = "5";
# synthetic {" # synthetic {"
# <?xml version="1.0" encoding="utf-8"?> # <?xml version="1.0" encoding="utf-8"?>
# <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" # <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
# "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> # "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
# <html> # <html>
# <head> # <head>
# <title>"} + obj.status + " " + obj.response + {"</title> # <title>"} + obj.status + " " + obj.response + {"</title>
# </head> # </head>
# <body> # <body>
# <h1>Error "} + obj.status + " " + obj.response + {"</h1> # <h1>Error "} + obj.status + " " + obj.response + {"</h1>
# <p>"} + obj.response + {"</p> # <p>"} + obj.response + {"</p>
# <h3>Guru Meditation:</h3> # <h3>Guru Meditation:</h3>
# <p>XID: "} + req.xid + {"</p> # <p>XID: "} + req.xid + {"</p>
# <hr> # <hr>
# <p>Varnish cache server</p> # <p>Varnish cache server</p>
# </body> # </body>
# </html> # </html>
# "}; # "};
# return (deliver); # return (deliver);
# } # }
# #
# sub vcl_init { # sub vcl_init {
# return (ok); # return (ok);
# } # }
# #
# sub vcl_fini { # sub vcl_fini {
# return (ok); # return (ok);
# } # }
   
import logging import logging
   
import ckan.plugins as plugins import ckan.plugins as plugins
import ckan.lib as lib import ckan.lib as lib
import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_dictize as model_dictize
import ckan.plugins.toolkit as tk import ckan.plugins.toolkit as tk
import ckan.model as model import ckan.model as model
from pylons import config from pylons import config
   
#parse the activity feed for last active non-system user #parse the activity feed for last active non-system user
def get_last_active_user(id): def get_last_active_user(id):
system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')}) system_user = lib.helpers.get_action('user_show',{'id': config.get('ckan.site_id', 'ckan_site_user')})
user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']] user_list = [x for x in lib.helpers.get_action('package_activity_list',{'id':id}) if x['user_id'] != system_user['id']]
user = None user = None
if len(user_list) > 0: if len(user_list) > 0:
user = user_list[0].get('user_id', None) user = user_list[0].get('user_id', None)
if user is None: if user is None:
return system_user return system_user
else: else:
return lib.helpers.get_action('user_show',{'id':user}) return lib.helpers.get_action('user_show',{'id':user})
   
# get user created datasets and those they have edited # get user created datasets and those they have edited
def get_user_datasets(user_dict): def get_user_datasets(user_dict):
created_datasets_list = user_dict['datasets'] created_datasets_list = user_dict['datasets']
active_datasets_list = [x['data']['package'] for x in active_datasets_list = [x['data']['package'] for x in
lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')]
return created_datasets_list + active_datasets_list return created_datasets_list + active_datasets_list
   
def datastore_search(context, data_dict):  
return {'success': True} # allow all datastore search  
   
class DataGovAuPlugin(plugins.SingletonPlugin, class DataGovAuPlugin(plugins.SingletonPlugin,
tk.DefaultDatasetForm): tk.DefaultDatasetForm):
'''An example IDatasetForm CKAN plugin. '''An example IDatasetForm CKAN plugin.
   
Uses a tag vocabulary to add a custom metadata field to datasets. Uses a tag vocabulary to add a custom metadata field to datasets.
   
''' '''
plugins.implements(plugins.IConfigurer, inherit=False) plugins.implements(plugins.IConfigurer, inherit=False)
plugins.implements(plugins.IDatasetForm, inherit=False) plugins.implements(plugins.IDatasetForm, inherit=False)
plugins.implements(plugins.ITemplateHelpers, inherit=False) plugins.implements(plugins.ITemplateHelpers, inherit=False)
plugins.implements(plugins.IAuthFunctions)  
   
   
def get_auth_functions(self):  
return {'datastore_search': datastore_search}  
   
   
def update_config(self, config): def update_config(self, config):
# Add this plugin's templates dir to CKAN's extra_template_paths, so # Add this plugin's templates dir to CKAN's extra_template_paths, so
# that CKAN will use this plugin's custom templates. # that CKAN will use this plugin's custom templates.
# here = os.path.dirname(__file__) # here = os.path.dirname(__file__)
# rootdir = os.path.dirname(os.path.dirname(here)) # rootdir = os.path.dirname(os.path.dirname(here))
   
tk.add_template_directory(config, 'templates') tk.add_template_directory(config, 'templates')
tk.add_public_directory(config, 'theme/public') tk.add_public_directory(config, 'theme/public')
tk.add_resource('theme/public', 'ckanext-datagovau') tk.add_resource('theme/public', 'ckanext-datagovau')
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json'
   
def get_helpers(self): def get_helpers(self):
return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets} return {'get_last_active_user': get_last_active_user, 'get_user_datasets': get_user_datasets}
   
def is_fallback(self): def is_fallback(self):
# Return True to register this plugin as the default handler for # Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin. # package types not handled by any other IDatasetForm plugin.
return True return True
   
def package_types(self): def package_types(self):
# This plugin doesn't handle any special package types, it just # This plugin doesn't handle any special package types, it just
# registers itself as the default (above). # registers itself as the default (above).
return [] return []
   
   
def create_package_schema(self): def create_package_schema(self):
schema = super(DataGovAuPlugin, self).create_package_schema() schema = super(DataGovAuPlugin, self).create_package_schema()
schema = self._modify_package_schema(schema) schema = self._modify_package_schema(schema)
return schema return schema
   
def update_package_schema(self): def update_package_schema(self):
schema = super(DataGovAuPlugin, self).update_package_schema() schema = super(DataGovAuPlugin, self).update_package_schema()
schema = self._modify_package_schema(schema) schema = self._modify_package_schema(schema)
return schema return schema
   
def show_package_schema(self): def show_package_schema(self):
schema = super(DataGovAuPlugin, self).show_package_schema() schema = super(DataGovAuPlugin, self).show_package_schema()
   
# Don't show vocab tags mixed in with normal 'free' tags # Don't show vocab tags mixed in with normal 'free' tags
# (e.g. on dataset pages, or on the search page) # (e.g. on dataset pages, or on the search page)
schema['tags']['__extras'].append(tk.get_converter('free_tags_only')) schema['tags']['__extras'].append(tk.get_converter('free_tags_only'))
   
# Add our custom_text field to the dataset schema. # Add our custom_text field to the dataset schema.
# ignore_missing == optional # ignore_missing == optional
# ignore_empty == mandatory but not for viewing # ignore_empty == mandatory but not for viewing
# !!! always convert_from_extras first # !!! always convert_from_extras first
schema.update({ schema.update({
'agency_program': [tk.get_converter('convert_from_extras'), 'agency_program': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_missing')], tk.get_validator('ignore_missing')],
'contact_point': [tk.get_converter('convert_from_extras'), 'contact_point': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'spatial_coverage': [tk.get_converter('convert_from_extras'), 'spatial_coverage': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'granularity': [tk.get_converter('convert_from_extras'), 'granularity': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'jurisdiction': [tk.get_converter('convert_from_extras'), 'jurisdiction': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'temporal_coverage': [tk.get_converter('convert_from_extras'), 'temporal_coverage': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'data_state': [tk.get_converter('convert_from_extras'), 'data_state': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')], tk.get_validator('ignore_empty')],
'update_freq': [tk.get_converter('convert_from_extras'), 'update_freq': [tk.get_converter('convert_from_extras'),
tk.get_validator('ignore_empty')] tk.get_validator('ignore_empty')]
}) })
return schema return schema
   
def _modify_package_schema(self, schema): def _modify_package_schema(self, schema):
# Add our custom_test metadata field to the schema, this one will use # Add our custom_test metadata field to the schema, this one will use
# convert_to_extras instead of convert_to_tags. # convert_to_extras instead of convert_to_tags.
# ignore_missing == optional # ignore_missing == optional
# not_empty == mandatory, enforced here while modifying # not_empty == mandatory, enforced here while modifying
   
schema.update({ schema.update({
'agency_program': [tk.get_validator('ignore_missing'), 'agency_program': [tk.get_validator('ignore_missing'),
tk.get_converter('convert_to_extras')], tk.get_converter('convert_to_extras')],
'contact_point': [tk.get_converter('convert_to_extras'), 'contact_point': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'spatial_coverage': [tk.get_converter('convert_to_extras'), 'spatial_coverage': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'granularity': [tk.get_converter('convert_to_extras'), 'granularity': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'jurisdiction': [tk.get_converter('convert_to_extras'), 'jurisdiction': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'temporal_coverage': [tk.get_converter('convert_to_extras'), 'temporal_coverage': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'data_state': [tk.get_converter('convert_to_extras'), 'data_state': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')], tk.get_validator('not_empty')],
'update_freq': [tk.get_converter('convert_to_extras'), 'update_freq': [tk.get_converter('convert_to_extras'),
tk.get_validator('not_empty')] tk.get_validator('not_empty')]
}) })
return schema return schema
   
# These methods just record how many times they're called, for testing # These methods just record how many times they're called, for testing
# purposes. # purposes.
# TODO: It might be better to test that custom templates returned by # TODO: It might be better to test that custom templates returned by
# these methods are actually used, not just that the methods get # these methods are actually used, not just that the methods get
# called. # called.
   
def setup_template_variables(self, context, data_dict): def setup_template_variables(self, context, data_dict):
return super(DataGovAuPlugin, self).setup_template_variables( return super(DataGovAuPlugin, self).setup_template_variables(
context, data_dict) context, data_dict)
   
def new_template(self): def new_template(self):
return super(DataGovAuPlugin, self).new_template() return super(DataGovAuPlugin, self).new_template()
   
def read_template(self): def read_template(self):
return super(DataGovAuPlugin, self).read_template() return super(DataGovAuPlugin, self).read_template()
   
def edit_template(self): def edit_template(self):
return super(DataGovAuPlugin, self).edit_template() return super(DataGovAuPlugin, self).edit_template()
   
def search_template(self): def search_template(self):
return super(DataGovAuPlugin, self).search_template() return super(DataGovAuPlugin, self).search_template()
   
def history_template(self): def history_template(self):
return super(DataGovAuPlugin, self).history_template() return super(DataGovAuPlugin, self).history_template()
   
def package_form(self): def package_form(self):
return super(DataGovAuPlugin, self).package_form() return super(DataGovAuPlugin, self).package_form()
   
   
{% ckan_extends %} {% ckan_extends %}
   
{% block links -%} {% block links -%}
{{ super() }} {{ super() }}
<link rel="schema.DCTERMS" href="http://purl.org/dc/terms/" /> <link rel="schema.DCTERMS" href="http://purl.org/dc/terms/" />
<link rel="schema.AGLSTERMS" href="http://www.agls.gov.au/agls/terms/" /> <link rel="schema.AGLSTERMS" href="http://www.agls.gov.au/agls/terms/" />
   
{% endblock -%} {% endblock -%}
   
{% block head_extras -%} {% block head_extras -%}
{{ super() }} {{ super() }}
<meta name="DCTERMS.Language" scheme="RFC3066" content="en" /> <meta name="DCTERMS.Language" scheme="RFC3066" content="en" />
<meta name="DCTERMS.Type" scheme="AGLSTERMS.Document" content="dataset" /> <meta name="DCTERMS.Type" scheme="AGLSTERMS.Document" content="dataset" />
<meta name="DCTERMS.Creator" scheme="AGLSTERMS.AglsAgent" content="{{ pkg['organization']['title'] }}" /> <meta name="DCTERMS.Creator" scheme="AGLSTERMS.AglsAgent" content="{{ pkg['organization']['title'] }}" />
<meta name="DCTERMS.Modified" scheme="DCTERMS.ISO8601" content="{{res['revision_timestamp']}}" /> <meta name="DCTERMS.Modified" scheme="DCTERMS.ISO8601" content="{{res['revision_timestamp']}}" />
<meta name="DCTERMS.Published" scheme="DCTERMS.ISO8601" content="{{res['created']}}" /> <meta name="DCTERMS.Published" scheme="DCTERMS.ISO8601" content="{{res['created']}}" />
<meta name="DCTERMS.Source.URI" content="{{pkg.url}}" />{% endif %} <meta name="DCTERMS.Source.URI" content="{{pkg.url}}" />
<meta name="DCTERMS.License" content="{{pkg['license_url']}}" /> <meta name="DCTERMS.License" content="{{pkg['license_url']}}" />
<meta name="DCTERMS.Coverage.Temporal" content="{{pkg.temporal_coverage}}" /> <meta name="DCTERMS.Coverage.Temporal" content="{{pkg.temporal_coverage}}" />
<meta name="DCTERMS.Coverage.Spatial" content="{{pkg.spatial_coverage}}" /> <meta name="DCTERMS.Coverage.Spatial" content="{{pkg.spatial_coverage}}" />
<meta name="AGLSTERMS.Jurisdiction" scheme="AGLSTERMS.AglsJuri" content="{{pkg.jurisdiction}}" /> <meta name="AGLSTERMS.Jurisdiction" scheme="AGLSTERMS.AglsJuri" content="{{pkg.jurisdiction}}" />
<meta name="DCAT.Theme" scheme="VO" content="{% for x in pkg.groups %}{{x['title']}},{% endfor %}" /> <meta name="DCAT.Theme" scheme="VO" content="{% for x in pkg.groups %}{{x['title']}},{% endfor %}" />
<meta name="DCAT.byteSize" content="{{res.get('size')}}" /> <meta name="DCAT.byteSize" content="{{res.get('size')}}" />
<meta name="DCAT.mediaType" content="{{res.get('mimetype')}}" /> <meta name="DCAT.mediaType" content="{{res.get('mimetype')}}" />
<meta name="DCTERMS.Identifier" content="{{res.get('url')}}" /> <meta name="DCTERMS.Identifier" content="{{res.get('url')}}" />
<meta name="DCTERMS.Title" content="{{res['name']}}" /> <meta name="DCTERMS.Title" content="{{res['name']}}" />
<meta name="DCTERMS.Description" scheme="" content="{{res['description']}}" /> <meta name="DCTERMS.Description" scheme="" content="{{res['description']}}" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="DCTERMS.Format" content="{{res.format}}" /> <meta name="DCTERMS.Format" content="{{res.format}}" />
<meta name="DCTERMS.File.Size" content="{{res.size}}" /> <meta name="DCTERMS.File.Size" content="{{res.size}}" />
   
{% endblock -%} {% endblock -%}