Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau
--- /dev/null
+++ b/admin/autoupdate.py
@@ -1,1 +1,75 @@
+import requests
+import ckanapi
+import csv
+import sys
+import psycopg2
+import json
+from subprocess import Popen, PIPE
+def updateresource(resource_id, dataset_id):
+ print ' --- '
+ ckan = ckanapi.RemoteCKAN(api_url,api_key)
+ #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org')
+ resource = ckan.action.resource_show(id=resource_id)
+ print 'updating '+resource['name']+'('+resource_id+', '+dataset_id+')'
+ print resource
+ url = resource['url']
+ #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT'
+ #etag='"1393206509.38-638"'
+ headers={}
+ if 'etag' in resource:
+ headers['If-None-Match'] = resource['etag']
+ if 'file_last_modified' in resource:
+ headers["If-Modified-Since"] = resource['file_last_modified']
+ print headers
+ r = requests.head(url, headers=headers)
+ if r.status_code == 304:
+ print 'not modified'
+ return
+ else:
+ print r.status_code
+ print r.headers
+ if 'last-modified' in r.headers:
+ resource['file_last_modified'] = r.headers['last-modified']
+ if 'etag' in r.headers:
+ resource['etag'] = r.headers['etag']
+ #save updated resource
+ print resource
+ result = ckan.call_action('resource_update',resource)
+ if resource['format'].lower() == 'shp' or resource['format'].lower() == 'kml':
+ print "geoingest!"
+ pargs= ['dga-spatialingestor.py', db_settings_json, api_url, api_key, dataset_id]
+ print pargs
+ p = Popen(pargs)#, stdout=PIPE, stderr=PIPE)
+ p.communicate()
+ else:
+ print "datapusher!"
+ # https://github.com/ckan/ckan/blob/master/ckanext/datapusher/logic/action.py#L19
+ ckan.action.datapusher_submit(resource_id=resource_id)
+
+if len(sys.argv) != 4:
+ print "autoupdate ingester. command line: postgis_url api_url api_key"
+ sys.exit(-1)
+else:
+ (path, db_settings_json, api_url, api_key) = sys.argv
+ db_settings = json.loads(db_settings_json)
+ datastore_db_settings = dict(db_settings)
+ datastore_db_settings['dbname'] = db_settings['datastore_dbname']
+ datastore_db_settings_json = json.dumps(datastore_db_settings)
+
+try:
+ conn = psycopg2.connect(dbname=db_settings['dbname'], user=db_settings['user'], password=db_settings['password'], host=db_settings['host'])
+except:
+ failure("I am unable to connect to the database.")
+# Open a cursor to perform database operations
+cur = conn.cursor()
+conn.set_isolation_level(0)
+cur.execute('select resource.id resource_id, package.id dataset_id from resource inner join resource_group on resource.resource_group_id = resource_group.id inner join package on resource_group.package_id = package.id where resource.extras like \'%"autoupdate": "active"%\';')
+row = cur.fetchone()
+while row is not None:
+ updateresource(row[0],row[1])
+ # process
+ row = cur.fetchone()
+cur.close()
+conn.close()
+
--- /dev/null
+++ b/admin/start.sh
@@ -1,1 +1,6 @@
+export NEW_RELIC_CONFIG_FILE="newrelic.ini"
+export VIRTUAL_ENV="/var/lib/ckan/dga/pyenv"
+export PATH="/var/lib/ckan/dga/pyenv:/var/lib/ckan/dga/pyenv/bin:$PATH"
+cd /var/lib/ckan/dga/pyenv/src/ckan
+newrelic-admin run-program paster serve development.ini
--- a/ckanext/datagovau/plugin.py
+++ b/ckanext/datagovau/plugin.py
@@ -22,6 +22,13 @@
filtered_dict[dataset['id']] = dataset
return filtered_dict.values()
+def get_related_dataset(related_id):
+ result = model.Session.execute("select dataset_id from related_dataset where related_id =\'"+related_id+"\' limit 1;").first()[0]
+ return result
+
+def related_create(context, data_dict=None):
+ return {'success': False, 'msg': 'No one is allowed to create related items'}
+
class DataGovAuPlugin(plugins.SingletonPlugin,
tk.DefaultDatasetForm):
'''An example IDatasetForm CKAN plugin.
@@ -31,6 +38,10 @@
'''
plugins.implements(plugins.IConfigurer, inherit=False)
plugins.implements(plugins.ITemplateHelpers, inherit=False)
+ plugins.implements(plugins.IAuthFunctions)
+
+ def get_auth_functions(self):
+ return {'related_create': related_create}
def update_config(self, config):
# Add this plugin's templates dir to CKAN's extra_template_paths, so
@@ -44,6 +55,6 @@
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json'
def get_helpers(self):
- return {'get_user_datasets': get_user_datasets}
+ return {'get_user_datasets': get_user_datasets, 'get_related_dataset': get_related_dataset}
--- a/ckanext/datagovau/templates/header.html
+++ b/ckanext/datagovau/templates/header.html
@@ -7,10 +7,11 @@
{{ h.build_nav_main(
('search', _('Datasets')),
('organizations_index', _('Organizations')),
- ('about', _('About'))
+ ('about', _('About')),
+ ('stats', _('Site Statistics'))
) }}
- <li><a href="//data.gov.au/stats">Site Statistics</a></li>
- <li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li>
+ <li><a href="/related">Use Cases</a></li>
+ <li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li>
{% endblock %}
</ul>
</nav>
--- a/ckanext/datagovau/templates/home/index.html
+++ b/ckanext/datagovau/templates/home/index.html
@@ -1,24 +1,18 @@
{% ckan_extends %}
- {% block home_search %}
-
-<div class="hero-secondary-inner">
-<script type="text/javascript" src="http://www.google.com/jsapi">
+{% block home_secondary_content %}
+<script type="text/javascript" src="//www.google.com/jsapi">
</script>
<script type="text/javascript">
google.load("feeds", "1") //Load Google Ajax Feed API (version 1)
</script>
- <div id="feeddiv">
-</div>
-</div>
-
-
+ <div id="feeddiv"></div>
<script type="text/javascript">
var feedcontainer=document.getElementById("feeddiv")
var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed"
-var feedlimit=4
+var feedlimit=3
var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>"
@@ -48,15 +42,6 @@
</script>
- <form class="module-content search-form" method="get" action="{% url_for controller='package', action='search' %}">
- <h3 class="heading">{{ _("Search Your Data") }}</h3>
- <div class="search-input control-group search-giant">
- <input type="text" class="search" name="q" value="{{ c.q }}" autocomplete="off" placeholder="{{ _('eg. Gold Prices') }}" />
- <button type="submit">
- <i class="icon-search"></i>
- <span>{{ _('Search') }}</span>
- </button>
- </div>
- </form>
+{{ super() }}
{% endblock %}
--- /dev/null
+++ b/ckanext/datagovau/templates/home/snippets/promoted.html
@@ -1,1 +1,17 @@
+{% set intro = g.site_intro_text %}
+<div class="module-content box">
+ <header>
+ {% if intro %}
+ {{ h.render_markdown(intro) }}
+ {% else %}
+ <h1 class="page-heading">{{ _("Welcome to CKAN") }}</h1>
+ <p>
+ {% trans %}This is a nice introductory paragraph about CKAN or the site
+ in general. We don't have any copy to go here yet but soon we will
+ {% endtrans %}
+ </p>
+ {% endif %}
+ </header>
+</div>
+
--- a/ckanext/datagovau/templates/home/snippets/search.html
+++ /dev/null
@@ -1,70 +1,1 @@
-<div class="hero-secondary-inner">
-<script type="text/javascript" src="http://www.google.com/jsapi">
-</script>
-<script type="text/javascript">
-google.load("feeds", "1") //Load Google Ajax Feed API (version 1)
-</script>
-
- <div id="feeddiv">
-</div>
-</div>
-
-
-
-<script type="text/javascript">
-
-var feedcontainer=document.getElementById("feeddiv")
-var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed"
-var feedlimit=4
-var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>"
-
-
-function rssfeedsetup(){
-var feedpointer=new google.feeds.Feed(feedurl) //Google Feed API method
-feedpointer.setNumEntries(feedlimit) //Google Feed API method
-feedpointer.load(displayfeed) //Google Feed API method
-}
-
-function displayfeed(result){
-if (!result.error){
-var thefeeds=result.feed.entries
-for (var i=0; i<thefeeds.length; i++) {
-rssoutput+="<div class='module module-shallow module-narrow module-dark info box'><h3><a href='" + thefeeds[i].link + "'>" + thefeeds[i].title + "</a></h3>"
-rssoutput+= " <font color='black'>Posted on " + new Date(thefeeds[i].publishedDate).toDateString() + "</font></div>"
-}
-rssoutput+=""
-feedcontainer.innerHTML=rssoutput
-}
-else
-alert("Error fetching feeds!")
-}
-
-window.onload=function(){
-rssfeedsetup()
-}
-
-</script>
-
-{% set tags = h.get_facet_items_dict('tags', limit=3) %}
-{% set placeholder = _('eg. Gold Prices') %}
-
-<div class="module module-search module-narrow module-shallow box">
- <form class="module-content search-form" method="get" action="{% url_for controller='package', action='search' %}">
- <h3 class="heading">{{ _("Search Your Data") }}</h3>
- <div class="search-input control-group search-giant">
- <input type="text" class="search" name="q" value="" autocomplete="off" placeholder="{{ placeholder }}" />
- <button type="submit">
- <i class="icon-search"></i>
- <span>{{ _('Search') }}</span>
- </button>
- </div>
- </form>
- <div class="tags">
- <h3>{{ _('Popular Tags') }}</h3>
- {% for tag in tags %}
- <a class="tag" href="{% url_for controller='package', action='search', tags=tag.name %}">{{ h.truncate(tag.display_name, 22) }}</a>
- {% endfor %}
- </div>
-</div>
-
--- /dev/null
+++ b/ckanext/datagovau/templates/home/snippets/stats.html
@@ -1,1 +1,78 @@
+{% set stats = h.get_site_statistics() %}
+
+<div class="box stats">
+ <div class="inner">
+ <h3>{{ _('{0} statistics').format(g.site_title) }}</h3>
+ <ul>
+ <li>
+ <a href="{{ h.url_for(controller='package', action='search') }}">
+ <b>{{ h.SI_number_span(stats.dataset_count) }}</b>
+ {{ _('dataset') if stats.dataset_count == 1 else _('datasets') }}
+ </a>
+ </li>
+ <li>
+ <a href="{{ h.url_for(controller='organization', action='index') }}">
+ <b>{{ h.SI_number_span(stats.organization_count) }}</b>
+ {{ _('organisation') if stats.organization_count == 1 else _('organisations') }}
+ </a>
+ </li>
+ <li>
+ <a href="{{ h.url_for(controller='group', action='index') }}">
+ <b>{{ h.SI_number_span(stats.group_count) }}</b>
+ {{ _('group') if stats.group_count == 1 else _('groups') }}
+ </a>
+ </li>
+ <!--<li>
+ <a href="{{ h.url_for(controller='related', action='dashboard') }}">
+ <b>{{ h.SI_number_span(stats.related_count) }}</b>
+ {{ _('related item') if stats.related_count == 1 else _('related items') }}
+ </a>
+ </li>-->
+ </ul>
+ </div>
+</div>
+
+ <div id="feeddiv">
+</div>
+
+<script type="text/javascript" src="//www.google.com/jsapi">
+</script>
+<script type="text/javascript">
+google.load("feeds", "1") //Load Google Ajax Feed API (version 1)
+</script>
+
+<script type="text/javascript">
+
+var feedcontainer=document.getElementById("feeddiv")
+var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed"
+var feedlimit=4
+var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>"
+
+
+function rssfeedsetup(){
+var feedpointer=new google.feeds.Feed(feedurl) //Google Feed API method
+feedpointer.setNumEntries(feedlimit) //Google Feed API method
+feedpointer.load(displayfeed) //Google Feed API method
+}
+
+function displayfeed(result){
+if (!result.error){
+var thefeeds=result.feed.entries
+for (var i=0; i<thefeeds.length; i++) {
+rssoutput+="<div class='module module-shallow module-narrow module-dark info box'><h3><a href='" + thefeeds[i].link + "'>" + thefeeds[i].title + "</a></h3>"
+rssoutput+= " <font color='black'>Posted on " + new Date(thefeeds[i].publishedDate).toDateString() + "</font></div>"
+}
+rssoutput+=""
+feedcontainer.innerHTML=rssoutput
+}
+else
+alert("Error fetching feeds!")
+}
+
+window.onload=function(){
+rssfeedsetup()
+}
+
+</script>
+
--- /dev/null
+++ b/ckanext/datagovau/templates/package/read_base.html
@@ -1,1 +1,10 @@
+{% ckan_extends %}
+{% block content_primary_nav %}
+ {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }}
+ {{ h.build_nav_icon('dataset_groups', _('Groups'), id=pkg.name) }}
+ {{ h.build_nav_icon('dataset_activity', _('Activity Stream'), id=pkg.name) }}
+ {{ h.build_nav_icon('related_list', _('Use Cases'), id=pkg.name) }}
+{% endblock %}
+
+
--- /dev/null
+++ b/ckanext/datagovau/templates/package/snippets/resource_form.html
@@ -1,1 +1,43 @@
+{% ckan_extends %}
+ {% block basic_fields_url %}
+ {% set is_upload = (data.url_type == 'upload') %}
+ {% set field_url='url' %}
+ {% set field_upload='upload' %}
+ {% set field_clear='clear_upload' %}
+ {% set is_upload_enabled=h.uploads_enabled() %}
+ {% set is_url=data.url and not is_upload %}
+ {% set upload_label=_('File') %}
+ {% set url_label=_('URL') %}
+
+ {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %}
+ {% set url_label = url_label or _('Image URL') %}
+ {% set upload_label = upload_label or _('Image') %}
+
+ {% if is_upload_enabled %}
+ <div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-is_upload="{{ 'true' if is_upload else 'false' }}"
+ data-module-field_url="{{ field_url }}" data-module-field_upload="{{ field_upload }}" data-module-field_clear="{{ field_clear }}" data-module-upload_label="{{ upload_label }}">
+ {% endif %}
+
+ {% call form.input(field_url, label=url_label, id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) %}
+<span id="autoupdate_form">
+ {% call form.select('autoupdate', label=_('Generate API from this Link'), options= [{'value': 'active', 'text': 'Active'}, {'value': 'inactive', 'text': 'Inactive'}], selected='Active', error=errors.autoupdate) %}
+<br/>
+Where a file is compatible with either CKAN or GeoServer we will attempt to make a functional end-point for this resource. The link provided above will also be checked for a new version based on the update frequency as set at the dataset level.
+</span>
+{% endcall %}
+{% endcall %}
+
+ {% if is_upload_enabled %}
+ {{ form.input(field_upload, label=upload_label, id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }}
+ {% if is_upload %}
+ {{form.checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+ {% endif %}
+ {% endif %}
+
+ {% if is_upload_enabled %}</div>{% endif %}
+
+
+ {% endblock %}
+
+
--- /dev/null
+++ b/ckanext/datagovau/templates/related/dashboard.html
@@ -1,1 +1,100 @@
+{% extends "page.html" %}
+{% set page = c.page %}
+{% set item_count = c.page.item_count %}
+
+{% block subtitle %}{{ _('Government Data Use Cases') }}{% endblock %}
+
+{% block breadcrumb_content %}
+ <li>{{ _('Government Data Use Cases') }}</li>
+{% endblock %}
+
+{% block primary_content %}
+ <article class="module">
+ <div class="module-content">
+ <h1 class="page-heading">
+ {% block page_heading %}{{ _('Government Data Use Cases') }}{% endblock %}
+ </h1>
+
+ {% block related_items %}
+ {% if item_count %}
+ {% trans first=page.first_item, last=page.last_item, item_count=item_count %}
+ <p>Showing items <strong>{{ first }} - {{ last }}</strong> of <strong>{{ item_count }}</strong> use cases found</p>
+ {% endtrans %}
+ {% elif c.filters.type %}
+ {% trans item_count=item_count %}
+ <p><strong>{{ item_count }}</strong> use cases found</p>
+ {% endtrans %}
+ {% else %}
+ <p class="empty">{{ _('There have been no use cases submitted yet.') }}
+ {% endif %}
+ {% endblock %}
+
+ {% block related_list %}
+ {% if page.items %}
+ {% snippet "related/snippets/related_list.html", related_items=page.items %}
+ {% endif %}
+ {% endblock %}
+ </div>
+
+ {% block page_pagination %}
+ {{ page.pager() }}
+ {% endblock %}
+ </article>
+{% endblock %}
+
+{% block secondary_content %}
+ <section class="module module-narrow module-shallow">
+ <h2 class="module-heading">{{ _('What are use cases?') }}</h2>
+ <div class="module-content">
+ {% trans %}
+ <p>Use Cases are any apps, articles, visualisations or ideas using datasets.</p>
+
+ <p>For example, it could be a custom visualisation, pictograph
+ or bar chart, an app using all or part of the data or even a news story
+ that references datasets from this site.</p>
+
+ <p> Send your ideas to <a href="mailto:data.gov@finance.gov.au">data.gov@finance.gov.au</a> with Title; Description; URL; Image URL; and Type: [API|Applications|Idea|News Article|Paper|Post|Visualisation]</p>
+ {% endtrans %}
+ </div>
+ </section>
+
+ <section class="module module-narrow module-shallow">
+ <h2 class="module-heading">{{ _('Filter Results') }}</h2>
+ <form action="" method="get" class="module-content form-inline form-narrow">
+ <input type='hidden' name='page' value='1'/>
+
+ <div class="control-group">
+ <label for="field-type">{{ _('Filter by type') }}</label>
+ <select id="field-type" name="type">
+ <option value="">{{ _('All') }}</option>
+ {% for option in c.type_options %}
+ <option value="{{ option.value }}"{% if c.filters.type == option.value %} selected="selected"{% endif %}>{{ option.text or option.value }}</option>
+ {% endfor %}
+ </select>
+ </div>
+
+ <div class="control-group">
+ <label for="field-sort">{{ _('Sort by') }}</label>
+ <select id="field-sort" name="sort">
+ <option value="">{{ _('Default') }}</option>
+ {% for option in c.sort_options %}
+ <option value="{{ option.value }}"{% if c.filters.sort == option.value %} selected="selected"{% endif %}>{{ option.text or option.value }}</option>
+ {% endfor %}
+ </select>
+ </div>
+
+ <div class="control-group">
+ <label for="field-featured" class="checkbox">
+ <input type="checkbox" id="field-featured" name="featured" {% if c.filters.get('featured') == 'on' %} checked="checked"{% endif %}></input>
+ {{ _('Only show featured items') }}
+ </label>
+ </div>
+
+ <div class="form-actions">
+ <button class="btn btn-primary">{{ _('Apply') }}</button>
+ </div>
+ </form>
+ </section>
+{% endblock %}
+
--- /dev/null
+++ b/ckanext/datagovau/templates/related/snippets/related_item.html
@@ -1,1 +1,48 @@
+{#
+Displays a single related item.
+related - The related item dict.
+pkg_id - The id of the owner package. If present the edit button will be
+ displayed.
+
+Example:
+
+
+
+#}
+{% set placeholder_map = {
+'application': h.url_for_static('/base/images/placeholder-application.png')
+} %}
+{% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %}
+<li class="related-item media-item" data-module="related-item">
+ <img src="{{ related.image_url or placeholder_map[related.type] or h.url_for_static('/base/images/placeholder-image.png') }}" alt="{{ related.title }}" class="media-image">
+ <h3 class="media-heading">{{ related.title }}</h3>
+ {% if related.description %}
+ <div class="prose">
+{{ h.render_markdown(related.description) }}
+</div>
+ {% endif %}
+ {% if h.check_access('package_show',{"id":h.get_related_dataset(related.id)}) %}
+<small>Using dataset: {{ h.get_action('package_show',{"id":h.get_related_dataset(related.id)}).title }}</small>
+ {% endif %}
+
+ <a class="media-view" href="{{ related.url }}" target="_blank" title="{{ tooltip }}">
+ <span>{{ tooltip }}</span>
+ <span class="banner">
+ {%- if related.type == 'application' -%}
+ app
+ {%- elif related.type == 'visualization' -%}
+ viz
+ {%- else -%}
+ {{ related.type | replace('news_', '') }}
+ {%- endif -%}
+ </span>
+ </a>
+ {% if pkg_id %}
+ {{ h.nav_link(_('Edit'), controller='related', action='edit', id=pkg_id, related_id=related.id, class_='btn btn-primary btn-small media-edit') }}
+ {% endif %}
+</li>
+{% if position is divisibleby 3 %}
+ <li class="clearfix js-hide"></li>
+{% endif %}
+