From: Alex Sadleir Date: Sun, 25 May 2014 11:26:32 +0000 Subject: Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=34265c9425b227190cd6dfa0f496299060e83915 --- Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau --- --- a/admin/autoupdate.py +++ b/admin/autoupdate.py @@ -1,18 +1,17 @@ import requests import ckanapi -# copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; -# f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" -import fileinput import csv +import sys +import psycopg2 +import json +from subprocess import Popen, PIPE -def updateresource(resource_id): - url = 'http://data.disclosurelo.gs' - api_key = '' - db_credentials = '' - print resource_id - ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') +def updateresource(resource_id, dataset_id): + print ' --- ' + ckan = ckanapi.RemoteCKAN(api_url,api_key) #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') resource = ckan.action.resource_show(id=resource_id) + print 'updating '+resource['name']+'('+resource_id+', '+dataset_id+')' print resource url = resource['url'] #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' @@ -22,6 +21,7 @@ headers['If-None-Match'] = resource['etag'] if 'file_last_modified' in resource: headers["If-Modified-Since"] = resource['file_last_modified'] + print headers r = requests.head(url, headers=headers) if r.status_code == 304: print 'not modified' @@ -34,20 +34,42 @@ if 'etag' in r.headers: resource['etag'] = r.headers['etag'] #save updated resource -# result = ckan.action.resource_update(id,resource) - if 'format' == 'shp': - print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id'] + print resource + result = ckan.call_action('resource_update',resource) + if resource['format'].lower() == 'shp' or resource['format'].lower() == 'kml': + print "geoingest!" + pargs= ['dga-spatialingestor.py', db_settings_json, api_url, api_key, dataset_id] + print pargs + p = Popen(pargs)#, stdout=PIPE, stderr=PIPE) + p.communicate() else: - ckan.action.datapusher_submit(resource_id=id) + print "datapusher!" + # https://github.com/ckan/ckan/blob/master/ckanext/datapusher/logic/action.py#L19 + ckan.action.datapusher_submit(resource_id=resource_id) -if len(sys.argv) != 5: - print "spatial ingester. command line: postgis_url api_url api_key" - sys.exit(errno.EACCES) +if len(sys.argv) != 4: + print "autoupdate ingester. command line: postgis_url api_url api_key" + sys.exit(-1) else: (path, db_settings_json, api_url, api_key) = sys.argv db_settings = json.loads(db_settings_json) + datastore_db_settings = dict(db_settings) + datastore_db_settings['dbname'] = db_settings['datastore_dbname'] + datastore_db_settings_json = json.dumps(datastore_db_settings) -for line in fileinput.input(): - row = csv.reader(line) - updateresource(line.replace("\n","")) +try: + conn = psycopg2.connect(dbname=db_settings['dbname'], user=db_settings['user'], password=db_settings['password'], host=db_settings['host']) +except: + failure("I am unable to connect to the database.") +# Open a cursor to perform database operations +cur = conn.cursor() +conn.set_isolation_level(0) +cur.execute('select resource.id resource_id, package.id dataset_id from resource inner join resource_group on resource.resource_group_id = resource_group.id inner join package on resource_group.package_id = package.id where resource.extras like \'%"autoupdate": "active"%\';') +row = cur.fetchone() +while row is not None: + updateresource(row[0],row[1]) + # process + row = cur.fetchone() +cur.close() +conn.close() --- a/ckanext/datagovau/plugin.py +++ b/ckanext/datagovau/plugin.py @@ -22,6 +22,10 @@ filtered_dict[dataset['id']] = dataset return filtered_dict.values() +def get_related_dataset(related_id): + result = model.Session.execute("select dataset_id from related_dataset where related_id =\'"+related_id+"\' limit 1;").first()[0] + return result + def related_create(context, data_dict=None): return {'success': False, 'msg': 'No one is allowed to create related items'} @@ -37,7 +41,7 @@ plugins.implements(plugins.IAuthFunctions) def get_auth_functions(self): - return {'rekated_create': related_create} + return {'related_create': related_create} def update_config(self, config): # Add this plugin's templates dir to CKAN's extra_template_paths, so @@ -51,6 +55,6 @@ # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' def get_helpers(self): - return {'get_user_datasets': get_user_datasets} + return {'get_user_datasets': get_user_datasets, 'get_related_dataset': get_related_dataset} --- a/ckanext/datagovau/templates/header.html +++ b/ckanext/datagovau/templates/header.html @@ -7,10 +7,11 @@ {{ h.build_nav_main( ('search', _('Datasets')), ('organizations_index', _('Organizations')), - ('about', _('About')) + ('about', _('About')), + ('stats', _('Site Statistics')) ) }} -
  • Site Statistics
    • -
  • Feedback/Request Data
    • +
  • Use Cases
    • +
  • Feedback/Request Data
    • {% endblock %} --- /dev/null +++ b/ckanext/datagovau/templates/package/read_base.html @@ -1,1 +1,10 @@ +{% ckan_extends %} +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_groups', _('Groups'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_activity', _('Activity Stream'), id=pkg.name) }} + {{ h.build_nav_icon('related_list', _('Use Cases'), id=pkg.name) }} +{% endblock %} + + --- /dev/null +++ b/ckanext/datagovau/templates/related/dashboard.html @@ -1,1 +1,100 @@ +{% extends "page.html" %} +{% set page = c.page %} +{% set item_count = c.page.item_count %} + +{% block subtitle %}{{ _('Government Data Use Cases') }}{% endblock %} + +{% block breadcrumb_content %} +
  • {{ _('Government Data Use Cases') }}
    • +{% endblock %} + +{% block primary_content %} +
    +
    +

    + {% block page_heading %}{{ _('Government Data Use Cases') }}{% endblock %} +

    + + {% block related_items %} + {% if item_count %} + {% trans first=page.first_item, last=page.last_item, item_count=item_count %} +

    Showing items {{ first }} - {{ last }} of {{ item_count }} use cases found

    + {% endtrans %} + {% elif c.filters.type %} + {% trans item_count=item_count %} +

    {{ item_count }} use cases found

    + {% endtrans %} + {% else %} +

    {{ _('There have been no use cases submitted yet.') }} + {% endif %} + {% endblock %} + + {% block related_list %} + {% if page.items %} + {% snippet "related/snippets/related_list.html", related_items=page.items %} + {% endif %} + {% endblock %} +

    + + {% block page_pagination %} + {{ page.pager() }} + {% endblock %} +
    +{% endblock %} + +{% block secondary_content %} +
    +

    {{ _('What are use cases?') }}

    +
    + {% trans %} +

    Use Cases are any apps, articles, visualisations or ideas using datasets.

    + +

    For example, it could be a custom visualisation, pictograph + or bar chart, an app using all or part of the data or even a news story + that references datasets from this site.

    + +

    Send your ideas to data.gov@finance.gov.au with Title; Description; URL; Image URL; and Type: [API|Applications|Idea|News Article|Paper|Post|Visualisation]

    + {% endtrans %} +
    +
    + +
    +

    {{ _('Filter Results') }}

    +
    + + +
    + + +
    + +
    + + +
    + +
    + +
    + +
    + +
    +
    +
    +{% endblock %} + --- /dev/null +++ b/ckanext/datagovau/templates/related/snippets/related_item.html @@ -1,1 +1,48 @@ +{# +Displays a single related item. +related - The related item dict. +pkg_id - The id of the owner package. If present the edit button will be + displayed. + +Example: + + + +#} +{% set placeholder_map = { +'application': h.url_for_static('/base/images/placeholder-application.png') +} %} +{% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %} +
  • + {{ related.title }} +

    {{ related.title }}

    + {% if related.description %} +
    +{{ h.render_markdown(related.description) }} +
    + {% endif %} + {% if h.check_access('package_show',{"id":h.get_related_dataset(related.id)}) %} +Using dataset: {{ h.get_action('package_show',{"id":h.get_related_dataset(related.id)}).title }} + {% endif %} + + + {{ tooltip }} + + + {% if pkg_id %} + {{ h.nav_link(_('Edit'), controller='related', action='edit', id=pkg_id, related_id=related.id, class_='btn btn-primary btn-small media-edit') }} + {% endif %} +
    • +{% if position is divisibleby 3 %} +
  • +{% endif %} +