rdf refactor
rdf refactor

--- a/admin/data.gov.au
+++ b/admin/data.gov.au
@@ -19,6 +19,11 @@
                       ExpiresDefault "access plus 1 week"
           </FilesMatch>
 </IfModule>
+AddType application/octet-stream .woff
+<FilesMatch "\.(eot|otf|woff|ttf)$">
+  SetEnvIf Origin "^http(s)?://(.+\.)?(data.gov.au)$" origin_is=$0
+  Header set Access-Control-Allow-Origin %{origin_is}e env=origin_is
+</FilesMatch>
 
 	<Directory />
 		Options FollowSymLinks

--- a/admin/default.vcl
+++ b/admin/default.vcl
@@ -26,6 +26,9 @@
  if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") {
     return(lookup);
  }
+if (req.url ~ "/(..|.._..)/") {
+    set req.url = regsub(req.url, "/((?!js)..|.._..)/", "/");
+}
   if (req.http.Cookie) {
     set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing
 
@@ -34,6 +37,17 @@
     }
   }
 } 
+sub vcl_hash {
+     # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl
+     set req.url = regsub(req.url, "(?:(.com|.au))/((?!js)..|.._..)/", "/");
+     hash_data(req.url);
+     if (req.http.host) {
+         hash_data(req.http.host);
+     } else {
+         hash_data(server.ip);
+     }
+     return (hash);
+}
 sub vcl_deliver {
     if (!resp.http.Vary) {
         set resp.http.Vary = "Accept-Encoding";   

--- /dev/null
+++ b/admin/development.ini
@@ -1,1 +1,365 @@
-
+#
+# ckan - Pylons configuration
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+[DEFAULT]
+
+# Change debug to true when doing CKAN development, it enables Pylons'
+# interactive debugging tool, makes Fanstatic serve unminified JS and CSS
+# files, and enables CKAN templates' debugging features.
+#   
+# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT*
+# Debug mode will enable the interactive debugging tool, allowing ANYONE to
+# execute malicious code after an exception is raised.
+debug = False
+
+email_to = ddg@alert.links.com.au
+error_email_from = data.gov@finance.gov.au
+
+# The SMTP server to connect to when sending emails to users, with optional
+# port (default: 25). For example: lavabit.com, or smtp.gmail.com:587. To
+# connect to a local sendmail process enter 'localhost'.
+#smtp.server = localhost
+smtp.server = smtp.gmail.com:587
+
+# Whether or not to use STARTTLS when connecting to the SMTP server.
+#smtp.starttls = False
+smtp.starttls = True
+
+# The username and password to use to authenticate with the SMTP server.
+# Optional, if these are left commented-out then CKAN will try to send email
+# to the SMTP server without logging in first.
+#smtp.user = your_username@gmail.com
+#smtp.password = your_password
+smtp.user = datagovau@gmail.com
+smtp.password = ***
+googleanalytics.id = ***
+# The email address that emails sent by CKAN to users should appear to come
+# from. Optional (default: None, note that the SMTP server may insert its own
+# from address).
+smtp.mail_from = data.gov@finance.gov.au
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+use = egg:ckan
+ckan.tracking_enabled = true
+full_stack = true
+cache_dir = %(here)s/data
+beaker.session.key = ckan
+beaker.session.secret = ***
+app_instance_uuid = ***
+# Auth
+
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_organizations = false
+ckan.auth.user_delete_organizations = true
+ckan.auth.user_create_groups = false
+ckan.auth.user_delete_groups = true
+ckan.auth.create_user_via_api = true
+
+# List the names of CKAN extensions to activate.
+# Note: This line is required to be here for packaging, even if it is empty.
+# Note: Add ``pdf_preview`` to enable the resource preview for PDFs
+#		Add the ``resource_proxy`` plugin to enable resorce proxying and get around the same origin policy
+#       Add ``datastore`` to enable the CKAN DataStore extension
+ckan.plugins = qa disqus stats json_preview recline_preview datastore datastorer googleanalytics resource_proxy wms_preview kml_preview geojson_preview datagovau
+
+disqus.name = datagovau
+
+# If you'd like to fine-tune the individual locations of the cache data dirs
+# for the Cache data, or the Session saves, un-comment the desired settings
+# here:
+#beaker.cache.data_dir = %(here)s/data/cache
+#beaker.session.data_dir = %(here)s/data/sessions
+
+# Specify the database for SQLAlchemy to use:
+# * Postgres is currently required for a production CKAN deployment
+# * Sqlite (memory or file) can be used as a quick alternative for testing
+#sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
+sqlalchemy.url = ***
+#sqlalchemy.url = sqlite:///
+#sqlalchemy.url = sqlite:///%(here)s/somedb.db
+
+# Un-comment and specify the URLs for the DataStore database.
+# * Postgres is required
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+ckan.datastore.write_url = ***
+ckan.datastore.read_url = ***
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+
+# Location of RDF versions of datasets
+#rdf_packages = http://semantic.ckan.net/record/
+
+# Location of licenses group (defaults to cached local version of ckan group)
+#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json
+licenses_group_url = file://%(here)s/licenses.json
+
+# Dataset form to use
+package_form = standard
+
+# Hide certain extras fields from dataset read form:
+# package_hide_extras = for_search_index_only
+
+# API configuration
+#apikey_header_name = X-CKAN-API-Key
+
+## extra places to look for templates and public files (comma separated lists)
+## any templates/files found will override correspondingly named ones in
+## ckan/templates/ and ckan/public
+## (e.g. to override main layout template layout.html or add extra css files)
+# extra_template_paths = %(here)s/my-templates
+# extra_public_paths = %(here)s/my-public
+
+# Dataset form integration
+#package_edit_return_url = http://another.frontend/dataset/<NAME>
+#package_new_return_url = http://another.frontend/dataset/<NAME>
+
+
+# Turn on messaging with carrot, default to false
+#ckan.async_notifier = true
+# Messaging module used by carrot:
+# * pyamqplib - AMQP (e.g. for RabbitMQ)
+# * queue - native Python Queue (debugging and tests only)
+#carrot_messaging_library = pyamqplib
+
+## Perform search just using database (rather than use e.g. solr).
+## In this setup search is crude and limited .e.g no full-text search, no faceting ...
+## However, very useful for getting up and running quickly with CKAN 
+# ckan.simple_search = 1
+
+## Title of site (using in several places including templates and <title> tag
+ckan.site_title = data.gov.au
+
+## Logo image to use on the home page
+ckan.site_logo = /base/images/ckan-logo.png
+
+## Site tagline / description (used on front page)
+ckan.site_description = Australian Government Open Data Catalogue
+
+## Used in creating some absolute urls (such as rss feeds, css files) and 
+## dump filenames
+ckan.site_url = http://data.gov.au
+
+## Favicon (default is the CKAN software favicon)
+ckan.favicon = /images/icons/ckan.ico
+
+## The gravatar default to use.  This can be any of the pre-defined strings
+## as defined on http://en.gravatar.com/site/implement/images/ (e.g. "identicon"
+## or "mm").  Or it can be a url, e.g. "http://example.com/images/avatar.jpg"
+ckan.gravatar_default = identicon
+
+## Solr support
+solr_url = http://solr.data.gov.au:8983/solr
+
+## Automatic indexing. Make all changes immediately available via the search
+## after editing or creating a dataset. Default is true. If for some reason
+## you need the indexing to occur asynchronously, set this option to 0.
+# ckan.search.automatic_indexing = 1
+
+## An 'id' for the site (using, for example, when creating entries in a common search index) 
+## If not specified derived from the site_url
+ckan.site_id = data.gov.au
+
+## API url to use (e.g. in AJAX callbacks)
+## Enable if the API is at a different domain
+# ckan.api_url = http://www.ckan.net
+
+## html content to be inserted just before </head> tag (e.g. extra stylesheet)
+## NB: can use html e.g. <strong>blah</strong>
+## NB: can have multiline strings just indent following lines
+# ckan.template_head_end = <link rel="stylesheet" href="http://mysite.org/css/custom.css" type="text/css"> 
+
+## html content to be inserted just before </body> tag (e.g. google analytics code)
+## NB: can use html e.g. <strong>blah</strong>
+## NB: can have multiline strings just indent following lines
+# ckan.template_footer_end = 
+
+# These three settings (ckan.log_dir, ckan.dump_dir and ckan.backup_dir) are
+# all used in cron jobs, not in CKAN itself. CKAN logging is configured 
+# in the logging configuration below
+# Directory for logs (produced by cron scripts associated with ckan)
+ckan.log_dir = %(here)s/log
+# Directory for JSON/CSV dumps (must match setting in apache config)
+ckan.dump_dir = %(here)s/dump
+# Directory for SQL database backups
+ckan.backup_dir = %(here)s/backup
+
+# Default authorizations for new domain objects
+#ckan.default_roles.Package = {"visitor": ["reader"], "logged_in": ["reader"]}
+#ckan.default_roles.Group = {"visitor": ["reader"], "logged_in": ["reader"]}
+#ckan.default_roles.System = {"visitor": ["reader"], "logged_in": ["editor"]}
+#ckan.default_roles.AuthorizationGroup = {"visitor": ["reader"], "logged_in": ["reader"]}
+
+## Ckan public and private recaptcha keys [localhost]
+#ckan.recaptcha.publickey = 6LcL-94SAAAAAKCHKLIgCYNJczalehF4a3GHlPyR
+#ckan.recaptcha.privatekey = 6LcL-94SAAAAADCsq995yBsNl7lHEgJ3Ukqf4Ykq
+
+# Locale/languages
+ckan.locale_default = en_GB
+ckan.locales_offered = en_GB
+# Languages are grouped by percentage of strings in CKAN 1.8 translated
+# (those with 100% first, then those with >=80%, then >=50%, then <50%) and
+# within these groups roughly sorted by number of worldwide native speakers
+# according to Wikipedia.
+ckan.locale_order = en_GB pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv
+ckan.locales_filtered_out = en
+
+## Atom Feeds
+#
+# Settings for customising the metadata provided in
+# atom feeds.
+#
+# These settings are used to generate the <id> tags for both feeds
+# and entries. The unique <id>s are created following the method
+# outlined in http://www.taguri.org/  ie - they generate tagURIs, as specified
+# in http://tools.ietf.org/html/rfc4151#section-2.1 :
+#
+# <id>tag:thedatahub.org,2012:/feeds/group/933f3857-79fd-4beb-a835-c0349e31ce76</id>
+#
+# Each component has the corresponding settings:
+#
+#   "thedatahub.org" is ckan.feeds.authority_name
+#   "2012"           is ckan.feeds.date
+#
+
+# Leave blank to use the ckan.site_url config value, otherwise set to a
+# domain or email address that you own.  e.g. thedatahub.org or
+# admin@thedatahub.org
+ckan.feeds.authority_name =
+
+# Pick a date of the form "yyyy[-mm[-dd]]" during which the above domain was
+# owned by you.
+ckan.feeds.date = 2012
+
+# If not set, then the value in `ckan.site_id` is used.
+ckan.feeds.author_name =
+
+# If not set, then the value in `ckan.site_url` is used.
+ckan.feeds.author_link =
+
+## File Store
+#
+# CKAN allows users to upload files directly to file storage either on the local
+# file system or to online ‘cloud’ storage like Amazon S3 or Google Storage.
+#
+# If you are using local file storage, remember to set ckan.site_url.
+#
+# To enable cloud storage (Google or S3), first run: pip install boto
+#
+# @see http://docs.ckan.org/en/latest/filestore.html
+
+# 'Bucket' to use for file storage
+ckan.storage.bucket = ckandga
+
+# To enable local file storage:
+ofs.impl = pairtree
+ofs.storage_dir = /srv/ckan/dga
+
+# To enable Google cloud storage:
+#ofs.impl = google
+#ofs.gs_access_key_id =
+#ofs.gs_secret_access_key =
+
+# To enable S3 cloud storage:
+#ofs.impl = s3
+#ofs.aws_access_key_id = ....
+#ofs.aws_secret_access_key = ....
+
+
+## Previews
+#
+# Set the file types that should be previewed inline (e.g. images) or directly in an iframe.
+
+ckan.preview.direct = png jpg gif
+ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
+
+
+# Activity Streams
+#
+# Default maximum number of activities to show in an activity stream.
+# ckan.activity_list_limit = 31
+
+
+# Activity Streams Email Notifications
+#
+# Uncomment this line to enable activity streams email notifications.
+# You also need to setup a cron job to send the emails, see the documentation.
+#ckan.activity_streams_email_notifications = True
+
+# Email notifications for events older than this time delta will not be sent.
+# Accepted formats: '2 days', '14 days', '4:35:00' (hours, minutes, seconds),
+# '7 days, 3:23:34', etc.
+# ckan.email_notifications_since = 2 days
+
+ckan.cache_expires = 2592000
+#ckan.page_cache_enabled = True
+ckan.static_max_age = 2592000
+ckan.cache_enabled = True
+# DEBUGGING
+
+# ckan.debug_supress_header This option can be set to suppress the debug
+# information showing the controller and action recieving the request being
+# shown in the header.  Note: This info only shows if debug is set to true.
+ckan.debug_supress_header = false
+
+## ===================================
+## Extensions
+
+## Config option to enable the (1 day) cache for stats
+## Default (if not defined) is True as stats computations are intensive
+# ckanext.stats.cache_enabled = True
+
+# Logging configuration
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console, file
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console, file
+
+[logger_ckan]
+level = ERROR
+handlers = console, file
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = WARNING
+handlers = console, file
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[handler_file]
+class = logging.handlers.RotatingFileHandler
+formatter = generic
+level = NOTSET
+#args = ("ckan.log", "a", 20000000, 9)
+args = ("/var/log/ckan/dga/ckan.log", "a", 20000000, 9)
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
+

--- a/ckanext/datagovau/plugin.py
+++ b/ckanext/datagovau/plugin.py
@@ -26,9 +26,6 @@
 				lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')]
     return created_datasets_list + active_datasets_list
 
-def datastore_search(context, data_dict):
-        return {'success': True} # allow all datastore search
-
 class DataGovAuPlugin(plugins.SingletonPlugin,
                                 tk.DefaultDatasetForm):
     '''An example IDatasetForm CKAN plugin.
@@ -39,12 +36,6 @@
     plugins.implements(plugins.IConfigurer, inherit=False)
     plugins.implements(plugins.IDatasetForm, inherit=False)
     plugins.implements(plugins.ITemplateHelpers, inherit=False)
-    plugins.implements(plugins.IAuthFunctions)
-
-
-    def get_auth_functions(self):
-        return {'datastore_search': datastore_search}
-
 
     def update_config(self, config):
         # Add this plugin's templates dir to CKAN's extra_template_paths, so

--- a/ckanext/datagovau/templates/package/read.rdf
+++ b/ckanext/datagovau/templates/package/read.rdf
@@ -23,28 +23,20 @@
         <dct:description>${c.pkg_dict['notes']}</dct:description>
         <dct:issued>${c.pkg_dict['metadata_created']}</dct:issued>
         <dct:modified>${c.pkg_dict['metadata_modified']}</dct:modified>
-        <dct:language>en</dct:language>
 
 <py:choose>
   <py:when test="c.pkg_dict.get('license_url',None)">
-        <dct:license>${c.pkg_dict['license_url']}</dct:license>
+        <dct:license rdf:resource="${c.pkg_dict['license_url']}"/>
         <dct:rights rdf:resource="${c.pkg_dict['license_url']}"/>
   </py:when>
   <py:otherwise>
         <dct:license>${c.pkg_dict['license_id']}</dct:license>
-        <dct:rights rdf:resource="${c.pkg_dict['license_id']}"/>
+        <dct:rights>"${c.pkg_dict['license_id']}"</dct:rights>
   </py:otherwise>
 </py:choose>
         <py:for each="tag_dict in c.pkg_dict.get('tags',[])">
             <dcat:keyword>${ tag_dict["name"] }</dcat:keyword>
         </py:for>
-        <py:for each="group_dict in c.pkg_dict.get('groups',[])">
-            <dcat:theme>${ group_dict["title"] }</dcat:theme>
-        </py:for>
-
-        <foaf:homepage
-                rdf:resource="${ h.url_for(controller='package',action='read',id=c.pkg_dict['name'], qualified=True)}"/>
-        <rdfs:label>${c.pkg_dict['name']}</rdfs:label>
 
 
         <py:for each="rsc_dict in c.pkg_dict['resources']">
@@ -69,6 +61,9 @@
                 </dcat:Distribution>
             </dcat:distribution>
         </py:for>
+
+
+<!-- data.gov.au specific stuff below this line -->
         <dct:publisher py:if="c.pkg_dict.get('organization', None)">
             <rdf:Description>
                 <foaf:name>${ c.pkg_dict['organization']['title'] }</foaf:name>
@@ -87,10 +82,26 @@
             </rdf:Description>
         </dct:contributor>
 
-
-        <foaf:homepage py:if="c.pkg_dict.get('url')">${c.pkg_dict.get('url')}</foaf:homepage>
+        <py:for each="extra_dict in c.pkg_dict.get('extras',[])">
+            <dct:relation>
+                <rdf:Description>
+                    <rdfs:label>${extra_dict.get('key','')}</rdfs:label>
+                    <rdf:value>${extra_dict.get('value','')}</rdf:value>
+                </rdf:Description>
+            </dct:relation>
+        </py:for>
+        <dct:language>en</dct:language>
+        <foaf:homepage
+                rdf:resource="${ h.url_for(controller='package',action='read',id=c.pkg_dict['name'], qualified=True)}"/>
         <dcat:contactPoint py:if="c.pkg_dict.get('contact_point')">${c.pkg_dict.contact_point }</dcat:contactPoint>
+<py:choose>
+  <py:when test="c.pkg_dict.get('spatial',None)">
+        <dct:spatial py:if="c.pkg_dict.get('spatial')">${ c.pkg_dict.spatial }</dct:spatial>
+  </py:when>
+  <py:otherwise>
         <dct:spatial py:if="c.pkg_dict.get('spatial_coverage')">${ c.pkg_dict.spatial_coverage }</dct:spatial>
+  </py:otherwise>
+</py:choose>
         <aglsterms:AglsJuri py:if="c.pkg_dict.get('jurisdiction')">${ c.pkg_dict.jurisdiction }</aglsterms:AglsJuri>
         <dct:temporal py:if="c.pkg_dict.get('temporal_coverage')">${ c.pkg_dict.get('temporal_coverage') }</dct:temporal>
         <dct:relation py:if="c.pkg_dict.get('data_state')">
@@ -117,14 +128,6 @@
                 <rdf:value>${ c.pkg_dict.get('granularity') }</rdf:value>
             </rdf:Description>
         </dct:relation>
-        <py:for each="extra_dict in c.pkg_dict.get('extras',[])">
-            <dct:relation>
-                <rdf:Description>
-                    <rdfs:label>${extra_dict.get('key','')}</rdfs:label>
-                    <rdf:value>${extra_dict.get('value','')}</rdf:value>
-                </rdf:Description>
-            </dct:relation>
-        </py:for>
     </dcat:Dataset>
 </rdf:RDF>
 

--- a/ckanext/datagovau/templates/package/resource_read.html
+++ b/ckanext/datagovau/templates/package/resource_read.html
@@ -14,7 +14,7 @@
 <meta name="DCTERMS.Creator" scheme="AGLSTERMS.AglsAgent" content="{{ pkg['organization']['title'] }}" />
 <meta name="DCTERMS.Modified" scheme="DCTERMS.ISO8601" content="{{res['revision_timestamp']}}" />
 <meta name="DCTERMS.Published" scheme="DCTERMS.ISO8601" content="{{res['created']}}" />
-<meta name="DCTERMS.Source.URI" content="{{pkg.url}}" />{% endif %}
+<meta name="DCTERMS.Source.URI" content="{{pkg.url}}" />
 <meta name="DCTERMS.License" content="{{pkg['license_url']}}" />
 <meta name="DCTERMS.Coverage.Temporal" content="{{pkg.temporal_coverage}}" />
 <meta name="DCTERMS.Coverage.Spatial" content="{{pkg.spatial_coverage}}" />