import requests | import requests |
import ckanapi | import ckanapi |
# copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; | # copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; |
# f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" | # f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" |
import fileinput | import fileinput |
import csv | import csv |
def updateresource(id): | def updateresource(resource_id): |
url = 'http://data.disclosurelo.gs' | url = 'http://data.disclosurelo.gs' |
api_key = '' | api_key = '' |
db_credentials = '' | db_credentials = '' |
print id | print resource_id |
ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') | ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') |
#ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') | #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') |
resource = ckan.action.resource_show(id=id) | resource = ckan.action.resource_show(id=resource_id) |
print resource | print resource |
url = resource['url'] | url = resource['url'] |
#last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' | #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' |
#etag='"1393206509.38-638"' | #etag='"1393206509.38-638"' |
headers={} | headers={} |
if 'etag' in resource: | if 'etag' in resource: |
headers['If-None-Match'] = resource['etag'] | headers['If-None-Match'] = resource['etag'] |
if 'file_last_modified' in resource: | if 'file_last_modified' in resource: |
headers["If-Modified-Since"] = resource['file_last_modified'] | headers["If-Modified-Since"] = resource['file_last_modified'] |
r = requests.head(url, headers=headers) | r = requests.head(url, headers=headers) |
if r.status_code == 304: | if r.status_code == 304: |
print 'not modified' | print 'not modified' |
return | return |
else: | else: |
print r.status_code | print r.status_code |
print r.headers | print r.headers |
if 'last-modified' in r.headers: | if 'last-modified' in r.headers: |
resource['file_last_modified'] = r.headers['last-modified'] | resource['file_last_modified'] = r.headers['last-modified'] |
if 'etag' in r.headers: | if 'etag' in r.headers: |
resource['etag'] = r.headers['etag'] | resource['etag'] = r.headers['etag'] |
#save updated resource | #save updated resource |
# result = ckan.action.resource_update(id,resource) | |
if 'format' == 'shp': | if 'format' == 'shp': |
print "geoingest!" | print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id'] |
else: | else: |
ckan.action.datapusher_submit(resource_id=id) | ckan.action.datapusher_submit(resource_id=id) |
# result = ckan.action.resource_update(id,resource) | |
if len(sys.argv) != 5: | |
print "spatial ingester. command line: postgis_url api_url api_key" | |
sys.exit(errno.EACCES) | |
else: | |
(path, db_settings_json, api_url, api_key) = sys.argv | |
db_settings = json.loads(db_settings_json) | |
for line in fileinput.input(): | for line in fileinput.input(): |
row = csv.reader(line) | row = csv.reader(line) |
updateresource(line.replace("\n","")) | updateresource(line.replace("\n","")) |
export NEW_RELIC_CONFIG_FILE="newrelic.ini" | |
export VIRTUAL_ENV="/var/lib/ckan/dga/pyenv" | |
export PATH="/var/lib/ckan/dga/pyenv:/var/lib/ckan/dga/pyenv/bin:$PATH" | |
cd /var/lib/ckan/dga/pyenv/src/ckan | |
newrelic-admin run-program paster serve development.ini | |
import logging | import logging |
import ckan.plugins as plugins | import ckan.plugins as plugins |
import ckan.lib as lib | import ckan.lib as lib |
import ckan.lib.dictization.model_dictize as model_dictize | import ckan.lib.dictization.model_dictize as model_dictize |
import ckan.plugins.toolkit as tk | import ckan.plugins.toolkit as tk |
import ckan.model as model | import ckan.model as model |
from pylons import config | from pylons import config |
from sqlalchemy import orm | from sqlalchemy import orm |
import ckan.model | import ckan.model |
# get user created datasets and those they have edited | # get user created datasets and those they have edited |
def get_user_datasets(user_dict): | def get_user_datasets(user_dict): |
created_datasets_list = user_dict['datasets'] | created_datasets_list = user_dict['datasets'] |
active_datasets_list = [x['data']['package'] for x in | active_datasets_list = [x['data']['package'] for x in |
lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] | lib.helpers.get_action('user_activity_list',{'id':user_dict['id']}) if x['data'].get('package')] |
raw_list = created_datasets_list + active_datasets_list | raw_list = created_datasets_list + active_datasets_list |
filtered_dict = {} | filtered_dict = {} |
for dataset in raw_list: | for dataset in raw_list: |
if dataset['id'] not in filtered_dict.keys(): | if dataset['id'] not in filtered_dict.keys(): |
filtered_dict[dataset['id']] = dataset | filtered_dict[dataset['id']] = dataset |
return filtered_dict.values() | return filtered_dict.values() |
def get_related_dataset(related_id): | |
result = model.Session.execute("select dataset_id from related_dataset where related_id =\'"+related_id+"\' limit 1;").first()[0] | |
return result | |
def related_create(context, data_dict=None): | |
return {'success': False, 'msg': 'No one is allowed to create related items'} | |
class DataGovAuPlugin(plugins.SingletonPlugin, | class DataGovAuPlugin(plugins.SingletonPlugin, |
tk.DefaultDatasetForm): | tk.DefaultDatasetForm): |
'''An example IDatasetForm CKAN plugin. | '''An example IDatasetForm CKAN plugin. |
Uses a tag vocabulary to add a custom metadata field to datasets. | Uses a tag vocabulary to add a custom metadata field to datasets. |
''' | ''' |
plugins.implements(plugins.IConfigurer, inherit=False) | plugins.implements(plugins.IConfigurer, inherit=False) |
plugins.implements(plugins.ITemplateHelpers, inherit=False) | plugins.implements(plugins.ITemplateHelpers, inherit=False) |
plugins.implements(plugins.IAuthFunctions) | |
def get_auth_functions(self): | |
return {'related_create': related_create} | |
def update_config(self, config): | def update_config(self, config): |
# Add this plugin's templates dir to CKAN's extra_template_paths, so | # Add this plugin's templates dir to CKAN's extra_template_paths, so |
# that CKAN will use this plugin's custom templates. | # that CKAN will use this plugin's custom templates. |
# here = os.path.dirname(__file__) | # here = os.path.dirname(__file__) |
# rootdir = os.path.dirname(os.path.dirname(here)) | # rootdir = os.path.dirname(os.path.dirname(here)) |
tk.add_template_directory(config, 'templates') | tk.add_template_directory(config, 'templates') |
tk.add_public_directory(config, 'theme/public') | tk.add_public_directory(config, 'theme/public') |
tk.add_resource('theme/public', 'ckanext-datagovau') | tk.add_resource('theme/public', 'ckanext-datagovau') |
# config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' | # config['licenses_group_url'] = 'http://%(ckan.site_url)/licenses.json' |
def get_helpers(self): | def get_helpers(self): |
return {'get_user_datasets': get_user_datasets} | return {'get_user_datasets': get_user_datasets, 'get_related_dataset': get_related_dataset} |
{% ckan_extends %} | {% ckan_extends %} |
{% block header_site_navigation %} | {% block header_site_navigation %} |
<nav class="section navigation"> | <nav class="section navigation"> |
<ul class="nav nav-pills"> | <ul class="nav nav-pills"> |
{% block header_site_navigation_tabs %} | {% block header_site_navigation_tabs %} |
{{ h.build_nav_main( | {{ h.build_nav_main( |
('search', _('Datasets')), | ('search', _('Datasets')), |
('organizations_index', _('Organizations')), | ('organizations_index', _('Organizations')), |
('about', _('About')) | ('about', _('About')), |
('stats', _('Site Statistics')) | |
) }} | ) }} |
<li><a href="//data.gov.au/stats">Site Statistics</a></li> | <li><a href="/related">Use Cases</a></li> |
<li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li> | <li><a href="https://datagovau.ideascale.com/">Feedback/Request Data</a></li> |
{% endblock %} | {% endblock %} |
</ul> | </ul> |
</nav> | </nav> |
{% endblock %} | {% endblock %} |
{% ckan_extends %} | {% ckan_extends %} |
{% block home_search %} | {% block home_secondary_content %} |
<script type="text/javascript" src="//www.google.com/jsapi"> | |
<div class="hero-secondary-inner"> | |
<script type="text/javascript" src="http://www.google.com/jsapi"> | |
</script> | </script> |
<script type="text/javascript"> | <script type="text/javascript"> |
google.load("feeds", "1") //Load Google Ajax Feed API (version 1) | google.load("feeds", "1") //Load Google Ajax Feed API (version 1) |
</script> | </script> |
<div id="feeddiv"> | <div id="feeddiv"></div> |
</div> | |
</div> | |
<script type="text/javascript"> | <script type="text/javascript"> |
var feedcontainer=document.getElementById("feeddiv") | var feedcontainer=document.getElementById("feeddiv") |
var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed" | var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed" |
var feedlimit=4 | var feedlimit=3 |
var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>" | var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>" |
function rssfeedsetup(){ | function rssfeedsetup(){ |
var feedpointer=new google.feeds.Feed(feedurl) //Google Feed API method | var feedpointer=new google.feeds.Feed(feedurl) //Google Feed API method |
feedpointer.setNumEntries(feedlimit) //Google Feed API method | feedpointer.setNumEntries(feedlimit) //Google Feed API method |
feedpointer.load(displayfeed) //Google Feed API method | feedpointer.load(displayfeed) //Google Feed API method |
} | } |
function displayfeed(result){ | function displayfeed(result){ |
if (!result.error){ | if (!result.error){ |
var thefeeds=result.feed.entries | var thefeeds=result.feed.entries |
for (var i=0; i<thefeeds.length; i++) { | for (var i=0; i<thefeeds.length; i++) { |
rssoutput+="<div class='module module-shallow module-narrow module-dark info box'><h3><a href='" + thefeeds[i].link + "'>" + thefeeds[i].title + "</a></h3>" | rssoutput+="<div class='module module-shallow module-narrow module-dark info box'><h3><a href='" + thefeeds[i].link + "'>" + thefeeds[i].title + "</a></h3>" |
rssoutput+= " <font color='black'>Posted on " + new Date(thefeeds[i].publishedDate).toDateString() + "</font></div>" | rssoutput+= " <font color='black'>Posted on " + new Date(thefeeds[i].publishedDate).toDateString() + "</font></div>" |
} | } |
rssoutput+="" | rssoutput+="" |
feedcontainer.innerHTML=rssoutput | feedcontainer.innerHTML=rssoutput |
} | } |
else | else |
alert("Error fetching feeds!") | alert("Error fetching feeds!") |
} | } |
window.onload=function(){ | window.onload=function(){ |
rssfeedsetup() | rssfeedsetup() |
} | } |
</script> | </script> |
<form class="module-content search-form" method="get" action="{% url_for controller='package', action='search' %}"> | {{ super() }} |
<h3 class="heading">{{ _("Search Your Data") }}</h3> | |
<div class="search-input control-group search-giant"> | |
<input type="text" class="search" name="q" value="{{ c.q }}" autocomplete="off" placeholder="{{ _('eg. Gold Prices') }}" /> | |
<button type="submit"> | |
<i class="icon-search"></i> | |
<span>{{ _('Search') }}</span> | |
</button> | |
</div> | |
</form> | |
{% endblock %} | {% endblock %} |
{% set intro = g.site_intro_text %} | |
<div class="module-content box"> | |
<header> | |
{% if intro %} | |
{{ h.render_markdown(intro) }} | |
{% else %} | |
<h1 class="page-heading">{{ _("Welcome to CKAN") }}</h1> | |
<p> | |
{% trans %}This is a nice introductory paragraph about CKAN or the site | |
in general. We don't have any copy to go here yet but soon we will | |
{% endtrans %} | |
</p> | |
{% endif %} | |
</header> | |
</div> | |
<div class="hero-secondary-inner"> | |
<script type="text/javascript" src="http://www.google.com/jsapi"> | |
</script> | |
<script type="text/javascript"> | |
google.load("feeds", "1") //Load Google Ajax Feed API (version 1) | |
</script> | |
<div id="feeddiv"> | |
</div> | |
</div> | |
<script type="text/javascript"> | |
var feedcontainer=document.getElementById("feeddiv") | |
var feedurl="http://www.finance.gov.au/taxonomy/term/1274/feed" | |
var feedlimit=4 | |
var rssoutput="<div class='module module-shallow module-narrow module-dark info box' style='color:black'><h2>Latest data.gov.au News</h2><i class='ckan-icon ckan-icon-feed'></i><a href='http://www.finance.gov.au/taxonomy/term/1274/feed/'> Subscribe to the blog </a></div>" | |
function rssfeedsetup(){ | |
var feedpointer=new google.feeds.Feed(feedurl) //Google Feed API method | |
feedpointer.setNumEntries(feedlimit) //Google Feed API method | |
feedpointer.load(displayfeed) //Google Feed API method | |
} | |
function displayfeed(result){ | |
if (!result.error){ | |
var thefeeds=result.feed.entries | |
for (var i=0; i<thefeeds.length; i++) { | |
rssoutput+="<div class='module module-shallow module-narrow module-dark info box'><h3><a href='" + thefeeds[i].link + "'>" + thefeeds[i].title + "</a></h3>" | |
rssoutput+= " <font color='black'>Posted on " + new Date(thefeeds[i].publishedDate).toDateString() + "</font></div>" | |
} | |
rssoutput+="" | |
feedcontainer.innerHTML=rssoutput | |
} | |
else | |
alert("Error fetching feeds!") | |
} | |
window.onload=function(){ | |
rssfeedsetup() | |
} | |
</script> | |
{% set tags = h.get_facet_items_dict('tags', limit=3) %} | |
{% set placeholder = _('eg. Gold Prices') %} | |
<div class="module module-search module-narrow module-shallow box"> | |
<form class="module-content search-form" method="get" action="{% url_for controller='package', action='search' %}"> | |
<h3 class="heading">{{ _("Search Your Data") }}</h3> | |
<div class="search-input control-group search-giant"> | |
<input type="text" class="search" name="q" value="" autocomplete="off" placeholder="{{ placeholder }}" /> | |
<button type="submit"> | |
<i class="icon-search"></i> | |
<span>{{ _('Search') }}</span> | |
</button> | |
</div> | |
</form> | |
<div class="tags"> | |
<h3>{{ _('Popular Tags') }}</h3> | |
{% for tag in tags %} | |
<a class="tag" href="{% url_for controller='package', action='search', tags=tag.name %}">{{ h.truncate(tag.display_name, 22) }}</a> | |
{% endfor %} | |
</div> | |
</div> | |
{% set stats = h.get_site_statistics() %} | |
<div class="box stats"> | |
<div class="inner"> | |
<h3>{{ _('{0} statistics').format(g.site_title) }}</h3> | |
<ul> | |
<li> | |
<a href="{{ h.url_for(controller='package', action='search') }}"> | |
<b>{{ h.SI_number_span(stats.dataset_count) }}</b> | |
{{ _('dataset') if stats.dataset_count == 1 else _('datasets') }} | |
</a> | |
</li> | |
<li> | |
<a href="{{ h.url_for(controller='organization', action='index') }}"> | |
<b>{{ h.SI_number_span(stats.organization_count) }}</b> | |
{{ _('organisation') if stats.organization_count == 1 else _('organisations') }} | |
</a> | |
</li> | |
<li> | |
<a href="{{ h.url_for(controller='group', action='index') }}"> | |
<b>{{ h.SI_number_span(stats.group_count) }}</b> | |
{{ _('group') if stats.group_count == 1 else _('groups') }} | |
</a> | |
</li> | |