From: Alex Sadleir Date: Fri, 04 Oct 2013 11:57:20 +0000 Subject: rdf refactor X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=3bbcfdf31b59526a38471de70dc32dd771567ed6 --- rdf refactor --- --- a/admin/data.gov.au +++ b/admin/data.gov.au @@ -19,6 +19,11 @@ ExpiresDefault "access plus 1 week" +AddType application/octet-stream .woff + + SetEnvIf Origin "^http(s)?://(.+\.)?(data.gov.au)$" origin_is=$0 + Header set Access-Control-Allow-Origin %{origin_is}e env=origin_is + Options FollowSymLinks --- a/admin/default.vcl +++ b/admin/default.vcl @@ -26,6 +26,9 @@ if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|woff|eot)$") { return(lookup); } +if (req.url ~ "/(..|.._..)/") { + set req.url = regsub(req.url, "/((?!js)..|.._..)/", "/"); +} if (req.http.Cookie) { set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1"); # removes all cookies named __utm? (utma, utmb...) - tracking thing @@ -34,6 +37,17 @@ } } } +sub vcl_hash { + # http://serverfault.com/questions/112531/ignoring-get-parameters-in-varnish-vcl + set req.url = regsub(req.url, "(?:(.com|.au))/((?!js)..|.._..)/", "/"); + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (hash); +} sub vcl_deliver { if (!resp.http.Vary) { set resp.http.Vary = "Accept-Encoding"; --- /dev/null +++ b/admin/development.ini @@ -1,1 +1,365 @@ - +# +# ckan - Pylons configuration +# +# The %(here)s variable will be replaced with the parent directory of this file +# +[DEFAULT] + +# Change debug to true when doing CKAN development, it enables Pylons' +# interactive debugging tool, makes Fanstatic serve unminified JS and CSS +# files, and enables CKAN templates' debugging features. +# +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* +# Debug mode will enable the interactive debugging tool, allowing ANYONE to +# execute malicious code after an exception is raised. +debug = False + +email_to = ddg@alert.links.com.au +error_email_from = data.gov@finance.gov.au + +# The SMTP server to connect to when sending emails to users, with optional +# port (default: 25). For example: lavabit.com, or smtp.gmail.com:587. To +# connect to a local sendmail process enter 'localhost'. +#smtp.server = localhost +smtp.server = smtp.gmail.com:587 + +# Whether or not to use STARTTLS when connecting to the SMTP server. +#smtp.starttls = False +smtp.starttls = True + +# The username and password to use to authenticate with the SMTP server. +# Optional, if these are left commented-out then CKAN will try to send email +# to the SMTP server without logging in first. +#smtp.user = your_username@gmail.com +#smtp.password = your_password +smtp.user = datagovau@gmail.com +smtp.password = *** +googleanalytics.id = *** +# The email address that emails sent by CKAN to users should appear to come +# from. Optional (default: None, note that the SMTP server may insert its own +# from address). +smtp.mail_from = data.gov@finance.gov.au + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:ckan +ckan.tracking_enabled = true +full_stack = true +cache_dir = %(here)s/data +beaker.session.key = ckan +beaker.session.secret = *** +app_instance_uuid = *** +# Auth + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_organizations = true +ckan.auth.user_create_groups = false +ckan.auth.user_delete_groups = true +ckan.auth.create_user_via_api = true + +# List the names of CKAN extensions to activate. +# Note: This line is required to be here for packaging, even if it is empty. +# Note: Add ``pdf_preview`` to enable the resource preview for PDFs +# Add the ``resource_proxy`` plugin to enable resorce proxying and get around the same origin policy +# Add ``datastore`` to enable the CKAN DataStore extension +ckan.plugins = qa disqus stats json_preview recline_preview datastore datastorer googleanalytics resource_proxy wms_preview kml_preview geojson_preview datagovau + +disqus.name = datagovau + +# If you'd like to fine-tune the individual locations of the cache data dirs +# for the Cache data, or the Session saves, un-comment the desired settings +# here: +#beaker.cache.data_dir = %(here)s/data/cache +#beaker.session.data_dir = %(here)s/data/sessions + +# Specify the database for SQLAlchemy to use: +# * Postgres is currently required for a production CKAN deployment +# * Sqlite (memory or file) can be used as a quick alternative for testing +#sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest +sqlalchemy.url = *** +#sqlalchemy.url = sqlite:/// +#sqlalchemy.url = sqlite:///%(here)s/somedb.db + +# Un-comment and specify the URLs for the DataStore database. +# * Postgres is required +#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore +#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore +ckan.datastore.write_url = *** +ckan.datastore.read_url = *** + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini + +# Location of RDF versions of datasets +#rdf_packages = http://semantic.ckan.net/record/ + +# Location of licenses group (defaults to cached local version of ckan group) +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +licenses_group_url = file://%(here)s/licenses.json + +# Dataset form to use +package_form = standard + +# Hide certain extras fields from dataset read form: +# package_hide_extras = for_search_index_only + +# API configuration +#apikey_header_name = X-CKAN-API-Key + +## extra places to look for templates and public files (comma separated lists) +## any templates/files found will override correspondingly named ones in +## ckan/templates/ and ckan/public +## (e.g. to override main layout template layout.html or add extra css files) +# extra_template_paths = %(here)s/my-templates +# extra_public_paths = %(here)s/my-public + +# Dataset form integration +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ + + +# Turn on messaging with carrot, default to false +#ckan.async_notifier = true +# Messaging module used by carrot: +# * pyamqplib - AMQP (e.g. for RabbitMQ) +# * queue - native Python Queue (debugging and tests only) +#carrot_messaging_library = pyamqplib + +## Perform search just using database (rather than use e.g. solr). +## In this setup search is crude and limited .e.g no full-text search, no faceting ... +## However, very useful for getting up and running quickly with CKAN +# ckan.simple_search = 1 + +## Title of site (using in several places including templates and tag +ckan.site_title = data.gov.au + +## Logo image to use on the home page +ckan.site_logo = /base/images/ckan-logo.png + +## Site tagline / description (used on front page) +ckan.site_description = Australian Government Open Data Catalogue + +## Used in creating some absolute urls (such as rss feeds, css files) and +## dump filenames +ckan.site_url = http://data.gov.au + +## Favicon (default is the CKAN software favicon) +ckan.favicon = /images/icons/ckan.ico + +## The gravatar default to use. This can be any of the pre-defined strings +## as defined on http://en.gravatar.com/site/implement/images/ (e.g. "identicon" +## or "mm"). Or it can be a url, e.g. "http://example.com/images/avatar.jpg" +ckan.gravatar_default = identicon + +## Solr support +solr_url = http://solr.data.gov.au:8983/solr + +## Automatic indexing. Make all changes immediately available via the search +## after editing or creating a dataset. Default is true. If for some reason +## you need the indexing to occur asynchronously, set this option to 0. +# ckan.search.automatic_indexing = 1 + +## An 'id' for the site (using, for example, when creating entries in a common search index) +## If not specified derived from the site_url +ckan.site_id = data.gov.au + +## API url to use (e.g. in AJAX callbacks) +## Enable if the API is at a different domain +# ckan.api_url = http://www.ckan.net + +## html content to be inserted just before </head> tag (e.g. extra stylesheet) +## NB: can use html e.g. <strong>blah</strong> +## NB: can have multiline strings just indent following lines +# ckan.template_head_end = <link rel="stylesheet" href="http://mysite.org/css/custom.css" type="text/css"> + +## html content to be inserted just before </body> tag (e.g. google analytics code) +## NB: can use html e.g. <strong>blah</strong> +## NB: can have multiline strings just indent following lines +# ckan.template_footer_end = + +# These three settings (ckan.log_dir, ckan.dump_dir and ckan.backup_dir) are +# all used in cron jobs, not in CKAN itself. CKAN logging is configured +# in the logging configuration below +# Directory for logs (produced by cron scripts associated with ckan) +ckan.log_dir = %(here)s/log +# Directory for JSON/CSV dumps (must match setting in apache config) +ckan.dump_dir = %(here)s/dump +# Directory for SQL database backups +ckan.backup_dir = %(here)s/backup + +# Default authorizations for new domain objects +#ckan.default_roles.Package = {"visitor": ["reader"], "logged_in": ["reader"]} +#ckan.default_roles.Group = {"visitor": ["reader"], "logged_in": ["reader"]} +#ckan.default_roles.System = {"visitor": ["reader"], "logged_in": ["editor"]} +#ckan.default_roles.AuthorizationGroup = {"visitor": ["reader"], "logged_in": ["reader"]} + +## Ckan public and private recaptcha keys [localhost] +#ckan.recaptcha.publickey = 6LcL-94SAAAAAKCHKLIgCYNJczalehF4a3GHlPyR +#ckan.recaptcha.privatekey = 6LcL-94SAAAAADCsq995yBsNl7lHEgJ3Ukqf4Ykq + +# Locale/languages +ckan.locale_default = en_GB +ckan.locales_offered = en_GB +# Languages are grouped by percentage of strings in CKAN 1.8 translated +# (those with 100% first, then those with >=80%, then >=50%, then <50%) and +# within these groups roughly sorted by number of worldwide native speakers +# according to Wikipedia. +ckan.locale_order = en_GB pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_filtered_out = en + +## Atom Feeds +# +# Settings for customising the metadata provided in +# atom feeds. +# +# These settings are used to generate the <id> tags for both feeds +# and entries. The unique <id>s are created following the method +# outlined in http://www.taguri.org/ ie - they generate tagURIs, as specified +# in http://tools.ietf.org/html/rfc4151#section-2.1 : +# +# <id>tag:thedatahub.org,2012:/feeds/group/933f3857-79fd-4beb-a835-c0349e31ce76</id> +# +# Each component has the corresponding settings: +# +# "thedatahub.org" is ckan.feeds.authority_name +# "2012" is ckan.feeds.date +# + +# Leave blank to use the ckan.site_url config value, otherwise set to a +# domain or email address that you own. e.g. thedatahub.org or +# admin@thedatahub.org +ckan.feeds.authority_name = + +# Pick a date of the form "yyyy[-mm[-dd]]" during which the above domain was +# owned by you. +ckan.feeds.date = 2012 + +# If not set, then the value in `ckan.site_id` is used. +ckan.feeds.author_name = + +# If not set, then the value in `ckan.site_url` is used. +ckan.feeds.author_link = + +## File Store +# +# CKAN allows users to upload files directly to file storage either on the local +# file system or to online ‘cloud’ storage like Amazon S3 or Google Storage. +# +# If you are using local file storage, remember to set ckan.site_url. +# +# To enable cloud storage (Google or S3), first run: pip install boto +# +# @see http://docs.ckan.org/en/latest/filestore.html + +# 'Bucket' to use for file storage +ckan.storage.bucket = ckandga + +# To enable local file storage: +ofs.impl = pairtree +ofs.storage_dir = /srv/ckan/dga + +# To enable Google cloud storage: +#ofs.impl = google +#ofs.gs_access_key_id = +#ofs.gs_secret_access_key = + +# To enable S3 cloud storage: +#ofs.impl = s3 +#ofs.aws_access_key_id = .... +#ofs.aws_secret_access_key = .... + + +## Previews +# +# Set the file types that should be previewed inline (e.g. images) or directly in an iframe. + +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json + + +# Activity Streams +# +# Default maximum number of activities to show in an activity stream. +# ckan.activity_list_limit = 31 + + +# Activity Streams Email Notifications +# +# Uncomment this line to enable activity streams email notifications. +# You also need to setup a cron job to send the emails, see the documentation. +#ckan.activity_streams_email_notifications = True + +# Email notifications for events older than this time delta will not be sent. +# Accepted formats: '2 days', '14 days', '4:35:00' (hours, minutes, seconds), +# '7 days, 3:23:34', etc. +# ckan.email_notifications_since = 2 days + +ckan.cache_expires = 2592000 +#ckan.page_cache_enabled = True +ckan.static_max_age = 2592000 +ckan.cache_enabled = True +# DEBUGGING + +# ckan.debug_supress_header This option can be set to suppress the debug +# information showing the controller and action recieving the request being +# shown in the header. Note: This info only shows if debug is set to true. +ckan.debug_supress_header = false + +## =================================== +## Extensions + +## Config option to enable the (1 day) cache for stats +## Default (if not defined) is True as stats computations are intensive +# ckanext.stats.cache_enabled = True + +# Logging configuration +[loggers] +keys = root, ckan, ckanext + +[handlers] +keys = console, file + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console, file + +[logger_ckan] +level = ERROR +handlers = console, file +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = WARNING +handlers = console, file +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[handler_file] +class = logging.handlers.RotatingFileHandler +formatter = generic +level = NOTSET +#args = ("ckan.log", "a", 20000000, 9) +args = ("/var/log/ckan/dga/ckan.log", "a", 20000000, 9) + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s + --- a/ckanext/datagovau/plugin.py +++ b/ckanext/datagovau/plugin.py @@ -26,9 +26,6 @@ lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] return created_datasets_list + active_datasets_list -def datastore_search(context, data_dict): - return {'success': True} # allow all datastore search - class DataGovAuPlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm): '''An example IDatasetForm CKAN plugin. @@ -39,12 +36,6 @@ plugins.implements(plugins.IConfigurer, inherit=False) plugins.implements(plugins.IDatasetForm, inherit=False) plugins.implements(plugins.ITemplateHelpers, inherit=False) - plugins.implements(plugins.IAuthFunctions) - - - def get_auth_functions(self): - return {'datastore_search': datastore_search} - def update_config(self, config): # Add this plugin's templates dir to CKAN's extra_template_paths, so --- a/ckanext/datagovau/templates/package/read.rdf +++ b/ckanext/datagovau/templates/package/read.rdf @@ -23,28 +23,20 @@ <dct:description>${c.pkg_dict['notes']}</dct:description> <dct:issued>${c.pkg_dict['metadata_created']}</dct:issued> <dct:modified>${c.pkg_dict['metadata_modified']}</dct:modified> - <dct:language>en</dct:language> <py:choose> <py:when test="c.pkg_dict.get('license_url',None)"> - <dct:license>${c.pkg_dict['license_url']}</dct:license> + <dct:license rdf:resource="${c.pkg_dict['license_url']}"/> <dct:rights rdf:resource="${c.pkg_dict['license_url']}"/> </py:when> <py:otherwise> <dct:license>${c.pkg_dict['license_id']}</dct:license> - <dct:rights rdf:resource="${c.pkg_dict['license_id']}"/> + <dct:rights>"${c.pkg_dict['license_id']}"</dct:rights> </py:otherwise> </py:choose> <py:for each="tag_dict in c.pkg_dict.get('tags',[])"> <dcat:keyword>${ tag_dict["name"] }</dcat:keyword> </py:for> - <py:for each="group_dict in c.pkg_dict.get('groups',[])"> - <dcat:theme>${ group_dict["title"] }</dcat:theme> - </py:for> - - <foaf:homepage - rdf:resource="${ h.url_for(controller='package',action='read',id=c.pkg_dict['name'], qualified=True)}"/> - <rdfs:label>${c.pkg_dict['name']}</rdfs:label> <py:for each="rsc_dict in c.pkg_dict['resources']"> @@ -69,6 +61,9 @@ </dcat:Distribution> </dcat:distribution> </py:for> + + +<!-- data.gov.au specific stuff below this line --> <dct:publisher py:if="c.pkg_dict.get('organization', None)"> <rdf:Description> <foaf:name>${ c.pkg_dict['organization']['title'] }</foaf:name> @@ -87,10 +82,26 @@ </rdf:Description> </dct:contributor> - - <foaf:homepage py:if="c.pkg_dict.get('url')">${c.pkg_dict.get('url')}</foaf:homepage> + <py:for each="extra_dict in c.pkg_dict.get('extras',[])"> + <dct:relation> + <rdf:Description> + <rdfs:label>${extra_dict.get('key','')}</rdfs:label> + <rdf:value>${extra_dict.get('value','')}</rdf:value> + </rdf:Description> + </dct:relation> + </py:for> + <dct:language>en</dct:language> + <foaf:homepage + rdf:resource="${ h.url_for(controller='package',action='read',id=c.pkg_dict['name'], qualified=True)}"/> <dcat:contactPoint py:if="c.pkg_dict.get('contact_point')">${c.pkg_dict.contact_point }</dcat:contactPoint> +<py:choose> + <py:when test="c.pkg_dict.get('spatial',None)"> + <dct:spatial py:if="c.pkg_dict.get('spatial')">${ c.pkg_dict.spatial }</dct:spatial> + </py:when> + <py:otherwise> <dct:spatial py:if="c.pkg_dict.get('spatial_coverage')">${ c.pkg_dict.spatial_coverage }</dct:spatial> + </py:otherwise> +</py:choose> <aglsterms:AglsJuri py:if="c.pkg_dict.get('jurisdiction')">${ c.pkg_dict.jurisdiction }</aglsterms:AglsJuri> <dct:temporal py:if="c.pkg_dict.get('temporal_coverage')">${ c.pkg_dict.get('temporal_coverage') }</dct:temporal> <dct:relation py:if="c.pkg_dict.get('data_state')"> @@ -117,14 +128,6 @@ <rdf:value>${ c.pkg_dict.get('granularity') }</rdf:value> </rdf:Description> </dct:relation> - <py:for each="extra_dict in c.pkg_dict.get('extras',[])"> - <dct:relation> - <rdf:Description> - <rdfs:label>${extra_dict.get('key','')}</rdfs:label> - <rdf:value>${extra_dict.get('value','')}</rdf:value> - </rdf:Description> - </dct:relation> - </py:for> </dcat:Dataset> </rdf:RDF> --- a/ckanext/datagovau/templates/package/resource_read.html +++ b/ckanext/datagovau/templates/package/resource_read.html @@ -14,7 +14,7 @@ <meta name="DCTERMS.Creator" scheme="AGLSTERMS.AglsAgent" content="{{ pkg['organization']['title'] }}" /> <meta name="DCTERMS.Modified" scheme="DCTERMS.ISO8601" content="{{res['revision_timestamp']}}" /> <meta name="DCTERMS.Published" scheme="DCTERMS.ISO8601" content="{{res['created']}}" /> -<meta name="DCTERMS.Source.URI" content="{{pkg.url}}" />{% endif %} +<meta name="DCTERMS.Source.URI" content="{{pkg.url}}" /> <meta name="DCTERMS.License" content="{{pkg['license_url']}}" /> <meta name="DCTERMS.Coverage.Temporal" content="{{pkg.temporal_coverage}}" /> <meta name="DCTERMS.Coverage.Spatial" content="{{pkg.spatial_coverage}}" />