From: Alex Sadleir Date: Thu, 05 Jun 2014 05:49:26 +0000 Subject: Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=201f2c6b0e1b908b77ba82803d341146d81cbbfc --- Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau --- --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,9 +1,94 @@ + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + localhost + 5050 + + + + + + + + + + + + + --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,36 +1,14 @@ - - - - - - - - - - + + + - - - - - - - - - - - - - - - - + - - - + + @@ -91,37 +68,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -130,7 +81,8 @@ - + + @@ -139,16 +91,8 @@ - - - - - - - - - - + + @@ -157,7 +101,8 @@ - + + @@ -166,7 +111,18 @@ - + + + + + + + + + + + + @@ -207,14 +163,16 @@ + - @@ -230,7 +188,7 @@ - + @@ -242,7 +200,6 @@ - @@ -271,7 +228,7 @@ @@ -288,6 +245,10 @@ + - + + - - - - - - + - - - - - + + - - @@ -478,6 +429,8 @@ - - - @@ -605,16 +553,16 @@ - - + + - + - - + + @@ -622,170 +570,143 @@ + + + - + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + - - + + - - - - - - - - + + - - + + - + + - - - - - - - - - - - - - - - + + - + + - - - - - - - - + + + + + + + + + + --- /dev/null +++ b/admin/autoupdate.py @@ -1,1 +1,90 @@ +#!/usr/bin/python +# coding=utf-8 +''' +auto update batch job for ckan + +1.0 26/05/2014 initial implementation +TODO +- archive files in filestore +- update frequency reduced based on dataset metadata +- emails on HTTP errors +''' + +import requests +import ckanapi +import csv +import sys +import psycopg2 +import json +from subprocess import Popen, PIPE + +def updateresource(resource_id, dataset_id): + print ' --- ' + ckan = ckanapi.RemoteCKAN(api_url,api_key) + #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') + resource = ckan.action.resource_show(id=resource_id) + print 'updating '+resource['name']+'('+resource_id+', '+dataset_id+')' + print resource + url = resource['url'] + #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' + #etag='"1393206509.38-638"' + headers={} + if 'etag' in resource: + headers['If-None-Match'] = resource['etag'] + if 'file_last_modified' in resource: + headers["If-Modified-Since"] = resource['file_last_modified'] + print headers + r = requests.head(url, headers=headers) + if r.status_code == 304: + print 'not modified' + return + else: + print r.status_code + print r.headers + if 'last-modified' in r.headers: + resource['file_last_modified'] = r.headers['last-modified'] + if 'etag' in r.headers: + resource['etag'] = r.headers['etag'] + #save updated resource + print resource + result = ckan.call_action('resource_update',resource) + if resource['format'].lower() == 'shp' or resource['format'].lower() == 'kml': + print "geoingest!" + pargs= ['dga-spatialingestor.py', db_settings_json, api_url, api_key, dataset_id] + print pargs + p = Popen(pargs)#, stdout=PIPE, stderr=PIPE) + p.communicate() + else: + print "datapusher!" + # https://github.com/ckan/ckan/blob/master/ckanext/datapusher/logic/action.py#L19 + ckan.action.datapusher_submit(resource_id=resource_id) + +if len(sys.argv) != 4: + print "autoupdate ingester. command line: postgis_url api_url api_key" + sys.exit(-1) +else: + (path, db_settings_json, api_url, api_key) = sys.argv + db_settings = json.loads(db_settings_json) + datastore_db_settings = dict(db_settings) + datastore_db_settings['dbname'] = db_settings['datastore_dbname'] + datastore_db_settings_json = json.dumps(datastore_db_settings) + +#until https://github.com/ckan/ckan/pull/1732 is merged, use database directly + +try: + conn = psycopg2.connect(dbname=db_settings['dbname'], user=db_settings['user'], password=db_settings['password'], host=db_settings['host']) +except: + failure("I am unable to connect to the database.") +# Open a cursor to perform database operations +cur = conn.cursor() +conn.set_isolation_level(0) +cur.execute('select resource.id resource_id, package.id dataset_id from resource inner join resource_group on resource.resource_group_id = resource_group.id inner join package on resource_group.package_id = package.id where resource.extras like \'%"autoupdate": "active"%\';') +row = cur.fetchone() +while row is not None: + updateresource(row[0],row[1]) + # process + row = cur.fetchone() +cur.close() +conn.close() + --- /dev/null +++ b/admin/start.sh @@ -1,1 +1,6 @@ +export NEW_RELIC_CONFIG_FILE="newrelic.ini" +export VIRTUAL_ENV="/var/lib/ckan/dga/pyenv" +export PATH="/var/lib/ckan/dga/pyenv:/var/lib/ckan/dga/pyenv/bin:$PATH" +cd /var/lib/ckan/dga/pyenv/src/ckan +newrelic-admin run-program paster serve development.ini --- a/ckanext-datagovau.iml +++ b/ckanext-datagovau.iml @@ -2,9 +2,7 @@ - - - + --- a/ckanext/datagovau/__init__.py +++ b/ckanext/datagovau/__init__.py @@ -1,1 +1,8 @@ +# this is a namespace package +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) --- a/ckanext/datagovau/plugin.py +++ b/ckanext/datagovau/plugin.py @@ -22,6 +22,13 @@ filtered_dict[dataset['id']] = dataset return filtered_dict.values() +def get_related_dataset(related_id): + result = model.Session.execute("select dataset_id from related_dataset where related_id =\'"+related_id+"\' limit 1;").first()[0] + return result + +def related_create(context, data_dict=None): + return {'success': False, 'msg': 'No one is allowed to create related items'} + class DataGovAuPlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm): '''An example IDatasetForm CKAN plugin. @@ -31,6 +38,10 @@ ''' plugins.implements(plugins.IConfigurer, inherit=False) plugins.implements(plugins.ITemplateHelpers, inherit=False) + plugins.implements(plugins.IAuthFunctions) + + def get_auth_functions(self): + return {'related_create': related_create} def update_config(self, config): # Add this plugin's templates dir to CKAN's extra_template_paths, so @@ -44,6 +55,6 @@ # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' def get_helpers(self): - return {'get_user_datasets': get_user_datasets} + return {'get_user_datasets': get_user_datasets, 'get_related_dataset': get_related_dataset} --- a/ckanext/datagovau/templates/header.html +++ b/ckanext/datagovau/templates/header.html @@ -1,4 +1,42 @@ {% ckan_extends %} + +{% block header_account_logged %} +{% if c.userobj.sysadmin %} +
  • + + Sysadmin settings + +
    • +{% endif %} +
  • + + {{ h.gravatar((c.userobj.email_hash if c and c.userobj else ''), size=22) }} + {{ c.userobj.display_name }} + +
    • +{% set new_activities = h.new_activities() %} +
  • + {% set notifications_tooltip = ngettext('Dashboard (%(num)d new item)', 'Dashboard (%(num)d new items)', new_activities) %} + + Dashboard + {{ new_activities }} + +
    • +{% block header_account_settings_link %} +
  • + + Edit settings + +
    • +{% endblock %} +{% block header_account_log_out_link %} +
  • + + Log out + +
    • +{% endblock %} +{% endblock %} {% block header_site_navigation %} --- a/ckanext/datagovau/templates/home/index.html +++ b/ckanext/datagovau/templates/home/index.html @@ -1,24 +1,18 @@ {% ckan_extends %} - {% block home_search %} - -
    - -
    -
    -
    - - +
    -
    -

    {{ _("Search Your Data") }}

    -
    - - -
    -
    +{{ super() }} {% endblock %} --- /dev/null +++ b/ckanext/datagovau/templates/home/snippets/promoted.html @@ -1,1 +1,17 @@ +{% set intro = g.site_intro_text %} +
    +
    + {% if intro %} + {{ h.render_markdown(intro) }} + {% else %} +

    {{ _("Welcome to CKAN") }}

    +

    + {% trans %}This is a nice introductory paragraph about CKAN or the site + in general. We don't have any copy to go here yet but soon we will + {% endtrans %} +

    + {% endif %} +
    +
    + --- a/ckanext/datagovau/templates/home/snippets/search.html +++ /dev/null @@ -1,70 +1,1 @@ -
    - - - -
    -
    -
    - - - - - -{% set tags = h.get_facet_items_dict('tags', limit=3) %} -{% set placeholder = _('eg. Gold Prices') %} - -
    -
    -

    {{ _("Search Your Data") }}

    -
    - - -
    -
    -
    -

    {{ _('Popular Tags') }}

    - {% for tag in tags %} - {{ h.truncate(tag.display_name, 22) }} - {% endfor %} -
    -
    - --- /dev/null +++ b/ckanext/datagovau/templates/home/snippets/stats.html @@ -1,1 +1,78 @@ +{% set stats = h.get_site_statistics() %} + +
    +
    +

    {{ _('{0} statistics').format(g.site_title) }}

    + +
    +
    + +
    +
    + + + + + + --- /dev/null +++ b/ckanext/datagovau/templates/package/read_base.html @@ -1,1 +1,10 @@ +{% ckan_extends %} +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_groups', _('Groups'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_activity', _('Activity Stream'), id=pkg.name) }} + {{ h.build_nav_icon('related_list', _('Use Cases'), id=pkg.name) }} +{% endblock %} + + --- /dev/null +++ b/ckanext/datagovau/templates/package/snippets/resource_form.html @@ -1,1 +1,43 @@ +{% ckan_extends %} + {% block basic_fields_url %} + {% set is_upload = (data.url_type == 'upload') %} + {% set field_url='url' %} + {% set field_upload='upload' %} + {% set field_clear='clear_upload' %} + {% set is_upload_enabled=h.uploads_enabled() %} + {% set is_url=data.url and not is_upload %} + {% set upload_label=_('File') %} + {% set url_label=_('URL') %} + + {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} + {% set url_label = url_label or _('Image URL') %} + {% set upload_label = upload_label or _('Image') %} + + {% if is_upload_enabled %} +
    + {% endif %} + + {% call form.input(field_url, label=url_label, id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) %} + + {% call form.select('autoupdate', label=_('Generate API from this Link'), options= [{'value': 'active', 'text': 'Active'}, {'value': 'inactive', 'text': 'Inactive'}], selected='Inactive', error=errors.autoupdate) %} +
    +Where a file is compatible with either CKAN or GeoServer we will attempt to make a functional end-point for this resource. The link provided above will also be checked for a new version based on the update frequency as set at the dataset level. +     +{% endcall %} +{% endcall %} + + {% if is_upload_enabled %} + {{ form.input(field_upload, label=upload_label, id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {% if is_upload %} + {{form.checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + + {% if is_upload_enabled %}
    {% endif %} + + + {% endblock %} + + --- /dev/null +++ b/ckanext/datagovau/templates/related/dashboard.html @@ -1,1 +1,100 @@ +{% extends "page.html" %} +{% set page = c.page %} +{% set item_count = c.page.item_count %} + +{% block subtitle %}{{ _('Government Data Use Cases') }}{% endblock %} + +{% block breadcrumb_content %} +
  • {{ _('Government Data Use Cases') }}
    • +{% endblock %} + +{% block primary_content %} +
    +
    +

    + {% block page_heading %}{{ _('Government Data Use Cases') }}{% endblock %} +

    + + {% block related_items %} + {% if item_count %} + {% trans first=page.first_item, last=page.last_item, item_count=item_count %} +

    Showin