import requests | import requests |
import ckanapi | import ckanapi |
# copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; | # copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; |
# f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" | # f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" |
import fileinput | import fileinput |
import csv | import csv |
def updateresource(id): | def updateresource(resource_id): |
url = 'http://data.disclosurelo.gs' | url = 'http://data.disclosurelo.gs' |
api_key = '' | api_key = '' |
db_credentials = '' | db_credentials = '' |
print id | print resource_id |
ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') | ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') |
#ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') | #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') |
resource = ckan.action.resource_show(id=id) | resource = ckan.action.resource_show(id=resource_id) |
print resource | print resource |
url = resource['url'] | url = resource['url'] |
#last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' | #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' |
#etag='"1393206509.38-638"' | #etag='"1393206509.38-638"' |
headers={} | headers={} |
if 'etag' in resource: | if 'etag' in resource: |
headers['If-None-Match'] = resource['etag'] | headers['If-None-Match'] = resource['etag'] |
if 'file_last_modified' in resource: | if 'file_last_modified' in resource: |
headers["If-Modified-Since"] = resource['file_last_modified'] | headers["If-Modified-Since"] = resource['file_last_modified'] |
r = requests.head(url, headers=headers) | r = requests.head(url, headers=headers) |
if r.status_code == 304: | if r.status_code == 304: |
print 'not modified' | print 'not modified' |
return | return |
else: | else: |
print r.status_code | print r.status_code |
print r.headers | print r.headers |
if 'last-modified' in r.headers: | if 'last-modified' in r.headers: |
resource['file_last_modified'] = r.headers['last-modified'] | resource['file_last_modified'] = r.headers['last-modified'] |
if 'etag' in r.headers: | if 'etag' in r.headers: |
resource['etag'] = r.headers['etag'] | resource['etag'] = r.headers['etag'] |
#save updated resource | #save updated resource |
# result = ckan.action.resource_update(id,resource) | |
if 'format' == 'shp': | if 'format' == 'shp': |
print "geoingest!" | print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id'] |
else: | else: |
ckan.action.datapusher_submit(resource_id=id) | ckan.action.datapusher_submit(resource_id=id) |
# result = ckan.action.resource_update(id,resource) | |
if len(sys.argv) != 5: | |
print "spatial ingester. command line: postgis_url api_url api_key" | |
sys.exit(errno.EACCES) | |
else: | |
(path, db_settings_json, api_url, api_key) = sys.argv | |
db_settings = json.loads(db_settings_json) | |
for line in fileinput.input(): | for line in fileinput.input(): |
row = csv.reader(line) | row = csv.reader(line) |
updateresource(line.replace("\n","")) | updateresource(line.replace("\n","")) |
import logging | import logging |
import ckan.plugins as plugins | import ckan.plugins as plugins |
import ckan.lib as lib | import ckan.lib as lib |
import ckan.lib.dictization.model_dictize as model_dictize | import ckan.lib.dictization.model_dictize as model_dictize |
import ckan.plugins.toolkit as tk | import ckan.plugins.toolkit as tk |
import ckan.model as model | import ckan.model as model |
from pylons import config | from pylons import config |
from sqlalchemy import orm | from sqlalchemy import orm |
import ckan.model | import ckan.model |
# get user created datasets and those they have edited | # get user created datasets and those they have edited |
def get_user_datasets(user_dict): | def get_user_datasets(user_dict): |
created_datasets_list = user_dict['datasets'] | created_datasets_list = user_dict['datasets'] |
active_datasets_list = [x['data']['package'] for x in | active_datasets_list = [x['data']['package'] for x in |
lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] | lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] |
raw_list = created_datasets_list + active_datasets_list | raw_list = created_datasets_list + active_datasets_list |
filtered_dict = {} | filtered_dict = {} |
for dataset in raw_list: | for dataset in raw_list: |
if dataset['id'] not in filtered_dict.keys(): | if dataset['id'] not in filtered_dict.keys(): |
filtered_dict[dataset['id']] = dataset | filtered_dict[dataset['id']] = dataset |
return filtered_dict.values() | return filtered_dict.values() |
def get_related_dataset(related_id): | |
result = model.Session.execute("select dataset_id from related_dataset where related_id =\'"+related_id+"\' limit 1;").first()[0] | |
return result | |
def related_create(context, data_dict=None): | def related_create(context, data_dict=None): |
return {'success': False, 'msg': 'No one is allowed to create related items'} | return {'success': False, 'msg': 'No one is allowed to create related items'} |
class DataGovAuPlugin(plugins.SingletonPlugin, | class DataGovAuPlugin(plugins.SingletonPlugin, |
tk.DefaultDatasetForm): | tk.DefaultDatasetForm): |
'''An example IDatasetForm CKAN plugin. | '''An example IDatasetForm CKAN plugin. |
Uses a tag vocabulary to add a custom metadata field to datasets. | Uses a tag vocabulary to add a custom metadata field to datasets. |
''' | ''' |
plugins.implements(plugins.IConfigurer, inherit=False) | plugins.implements(plugins.IConfigurer, inherit=False) |
plugins.implements(plugins.ITemplateHelpers, inherit=False) | plugins.implements(plugins.ITemplateHelpers, inherit=False) |
plugins.implements(plugins.IAuthFunctions) | plugins.implements(plugins.IAuthFunctions) |
def get_auth_functions(self): | def get_auth_functions(self): |
return {'rekated_create': related_create} | return {'related_create': related_create} |
def update_config(self, config): | def update_config(self, config): |
# Add this plugin's templates dir to CKAN's extra_template_paths, so | # Add this plugin's templates dir to CKAN's extra_template_paths, so |
# that CKAN will use this plugin's custom templates. | # that CKAN will use this plugin's custom templates. |
# here = os.path.dirname(__file__) | # here = os.path.dirname(__file__) |
# rootdir = os.path.dirname(os.path.dirname(here)) | # rootdir = os.path.dirname(os.path.dirname(here)) |
tk.add_template_directory(config, 'templates') | tk.add_template_directory(config, 'templates') |
tk.add_public_directory(config, 'theme/public') | tk.add_public_directory(config, 'theme/public') |
tk.add_resource('theme/public', 'ckanext-datagovau') | tk.add_resource('theme/public', 'ckanext-datagovau') |
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' | # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' |
def get_helpers(self): | def get_helpers(self): |
return {'get_user_datasets': get_user_datasets} | return {'get_user_datasets': get_user_datasets, 'get_related_dataset': get_related_dataset} |
{% ckan_extends %} | {% ckan_extends %} |
{% block header_site_navigation %} | {% block header_site_navigation %} |
<nav class="section navigation"> | <nav class="section navigation"> |
<ul class="nav nav-pills"> | <ul class="nav nav-pills"> |
{% block header_site_navigation_tabs %} | {% block header_site_navigation_tabs %} |
{{ h.build_nav_main( | {{ h.build_nav_main( |
('search', _('Datasets')), | ('search', _('Datasets')), |
('organizations_index', _('Organizations')), | ('organizations_index', _('Organizations')), |
('about', _('About')) | ('about', _('About')), |
('stats', _('Site Statistics')) | |
) }} | ) }} |
<li><a href="//data.gov.au/stats">Site Statistics</a></li> | <li><a href="/related">Use Cases</a></li> |
<li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li> | <li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li> |
{% endblock %} | {% endblock %} |
</ul> | </ul> |
</nav> | </nav> |
{% endblock %} | {% endblock %} |
{% ckan_extends %} | |
{% block content_primary_nav %} | |
{{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} | |
{{ h.build_nav_icon('dataset_groups', _('Groups'), id=pkg.name) }} | |
{{ h.build_nav_icon('dataset_activity', _('Activity Stream'), id=pkg.name) }} | |
{{ h.build_nav_icon('related_list', _('Use Cases'), id=pkg.name) }} | |
{% endblock %} | |
{% ckan_extends %} | {% ckan_extends %} |
{% block basic_fields_url %} | {% block basic_fields_url %} |
{% set is_upload = (data.url_type == 'upload') %} | {% set is_upload = (data.url_type == 'upload') %} |
{% set field_url='url' %} | {% set field_url='url' %} |
{% set field_upload='upload' %} | {% set field_upload='upload' %} |
{% set field_clear='clear_upload' %} | {% set field_clear='clear_upload' %} |
{% set is_upload_enabled=h.uploads_enabled() %} | {% set is_upload_enabled=h.uploads_enabled() %} |
{% set is_url=data.url and not is_upload %} | {% set is_url=data.url and not is_upload %} |
{% set upload_label=_('File') %} | {% set upload_label=_('File') %} |
{% set url_label=_('URL') %} | {% set url_label=_('URL') %} |
{% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} | {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} |
{% set url_label = url_label or _('Image URL') %} | {% set url_label = url_label or _('Image URL') %} |
{% set upload_label = upload_label or _('Image') %} | {% set upload_label = upload_label or _('Image') %} |
{% if is_upload_enabled %} | {% if is_upload_enabled %} |
<div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-is_upload="{{ 'true' if is_upload else 'false' }}" | <div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-is_upload="{{ 'true' if is_upload else 'false' }}" |
data-module-field_url="{{ field_url }}" data-module-field_upload="{{ field_upload }}" data-module-field_clear="{{ field_clear }}" data-module-upload_label="{{ upload_label }}"> | data-module-field_url="{{ field_url }}" data-module-field_upload="{{ field_upload }}" data-module-field_clear="{{ field_clear }}" data-module-upload_label="{{ upload_label }}"> |
{% endif %} | {% endif %} |
{% call form.input(field_url, label=url_label, id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) %} | {% call form.input(field_url, label=url_label, id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) %} |
<span id="autoupdate_form"> | <span id="autoupdate_form"> |
{% call form.select('autoupdate', label=_('Generate API from this Link'), options= [{'value': 'active', 'text': 'Active'}, {'value': 'inactive', 'text': 'Inactive'}], selected='Active', error=errors.autoupdate) %} | {% call form.select('autoupdate', label=_('Generate API from this Link'), options= [{'value': 'active', 'text': 'Active'}, {'value': 'inactive', 'text': 'Inactive'}], selected='Inactive', error=errors.autoupdate) %} |
<br/> | <br/> |
Where a file is compatible with either CKAN or GeoServer we will attempt to make a functional end-point for this resource. The link provided above will also be checked for a new version based on the update frequency as set at the dataset level. | Where a file is compatible with either CKAN or GeoServer we will attempt to make a functional end-point for this resource. The link provided above will also be checked for a new version based on the update frequency as set at the dataset level. |
</span> | </span> |
{% endcall %} | {% endcall %} |
{% endcall %} | {% endcall %} |
{% if is_upload_enabled %} | {% if is_upload_enabled %} |
{{ form.input(field_upload, label=upload_label, id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} | {{ form.input(field_upload, label=upload_label, id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} |
{% if is_upload %} | {% if is_upload %} |
{{form.checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} | {{form.checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} |
{% endif %} | {% endif %} |
{% endif %} | {% endif %} |
{% if is_upload_enabled %}</div>{% endif %} | {% if is_upload_enabled %}</div>{% endif %} |
{% endblock %} | {% endblock %} |
{% extends "page.html" %} | |
{% set page = c.page %} | |
{% set item_count = c.page.item_count %} | |
{% block subtitle %}{{ _('Government Data Use Cases') }}{% endblock %} | |
{% block breadcrumb_content %} | |
<li>{{ _('Government Data Use Cases') }}</li> | |
{% endblock %} | |
{% block primary_content %} | |
<article class="module"> | |
<div class="module-content"> | |
<h1 class="page-heading"> | |
{% block page_heading %}{{ _('Government Data Use Cases') }}{% endblock %} | |
</h1> | |
{% block related_items %} | |
{% if item_count %} | |
{% trans first=page.first_item, last=page.last_item, item_count=item_count %} | |
<p>Showing items <strong>{{ first }} - {{ last }}</strong> of <strong>{{ item_count }}</strong> use cases found</p> | |
{% endtrans %} | |
{% elif c.filters.type %} | |
{% trans item_count=item_count %} | |
<p><strong>{{ item_count }}</strong> use cases found</p> | |
{% endtrans %} | |
{% else %} | |
<p class="empty">{{ _('There have been no use cases submitted yet.') }} | |
{% endif %} | |
{% endblock %} | |
{% block related_list %} | |
{% if page.items %} | |
{% snippet "related/snippets/related_list.html", related_items=page.items %} | |
{% endif %} | |
{% endblock %} | |
</div> | |
{% block page_pagination %} | |
{{ page.pager() }} | |
{% endblock %} | |
</article> | |
{% endblock %} | |
{% block secondary_content %} | |
<section class="module module-narrow module-shallow"> | |
<h2 class="module-heading">{{ _('What are use cases?') }}</h2> | |
<div class="module-content"> | |
{% trans %} | |
<p>Use Cases are any apps, articles, visualisations or ideas using datasets.</p> | |
<p>For example, it could be a custom visualisation, pictograph | |
or bar chart, an app using all or part of the data or even a news story | |
that references datasets from this site.</p> | |
<p> Send your ideas to <a href="mailto:data.gov@finance.gov.au">data.gov@finance.gov.au</a> with Title; Description; URL; Image URL; and Type: [API|Applications|Idea|News Article|Paper|Post|Visualisation]</p> | |
{% endtrans %} | |
</div> | |
</section> | |
<section class="module module-narrow module-shallow"> | |
<h2 class="module-heading">{{ _('Filter Results') }}</h2> | |
<form action="" method="get" class="module-content form-inline form-narrow"> | |
<input type='hidden' name='page' value='1'/> | |
<div class="control-group"> | |
<label for="field-type">{{ _('Filter by type') }}</label> | |
<select id="field-type" name="type"> | |
<option value="">{{ _('All') }}</option> | |
{% for option in c.type_options %} | |
<option value="{{ option.value }}"{% if c.filters.type == option.value %} selected="selected"{% endif %}>{{ option.text or option.value }}</option> | |
{% endfor %} | |
</select> | |
</div> | |
<div class="control-group"> | |
<label for="field-sort">{{ _('Sort by') }}</label> | |
<select id="field-sort" name="sort"> | |
<option value="">{{ _('Default') }}</option> | |
{% for option in c.sort_options %} | |
<option value="{{ option.value }}"{% if c.filters.sort == option.value %} selected="selected"{% endif %}>{{ option.text or option.value }}</option> | |
{% endfor %} | |
</select> | |
</div> | |
<div class="control-group"> | |
<label for="field-featured" class="checkbox"> | |
<input type="checkbox" id="field-featured" name="featured" {% if c.filters.get('featured') == 'on' %} checked="checked"{% endif %}></input> | |
{{ _('Only show featured items') }} | |
</label> | |
</div> | |
<div class="form-actions"> | |
<button class="btn btn-primary">{{ _('Apply') }}</button> | |
</div> | |
</form> | |
</section> | |
{% endblock %} | |
{# | |
Displays a single related item. | |
related - The related item dict. | |
pkg_id - The id of the owner package. If present the edit button will be | |
displayed. | |
Example: | |
#} | |
{% set placeholder_map = { | |
'application': h.url_for_static('/base/images/placeholder-application.png') | |
} %} | |
{% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %} | |
<li class="related-item media-item" data-module="related-item"> | |
<img src="{{ related.image_url or placeholder_map[related.type] or h.url_for_static('/base/images/placeholder-image.png') }}" alt="{{ related.title }}" class="media-image"> | |
<h3 class="media-heading">{{ related.title }}</h3> | |
{% if related.description %} | |
<div class="prose"> | |
{{ h.render_markdown(related.description) }} | |
</div> | |
{% endif %} | |
{% if h.check_access('package_show',{"id":h.get_related_dataset(related.id)}) %} | |
<small>Using dataset: {{ h.get_action('package_show',{"id":h.get_related_dataset(related.id)}).title }}</small> | |
{% endif %} | |
<a class="media-view" href="{{ related.url }}" target="_blank" title="{{ tooltip }}"> |