import ckanclient |
import ckanclient |
import couchdb |
import couchdb |
from ckanclient import CkanApiError |
from ckanclient import CkanApiError |
import re |
import re |
|
|
|
|
class LoaderError(Exception): |
class LoaderError(Exception): |
pass |
pass |
|
|
# Instantiate the CKAN client. |
# Instantiate the CKAN client. |
#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') |
#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') |
api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' |
api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' |
ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', |
server = 'data.disclosurelo.gs' |
|
api_key = 'c30eb6f5-0f90-47e0-bf05-9b1b4e3a461a' |
|
server = 'ckan.data.gov.au' |
|
|
|
ckan = ckanclient.CkanClient(base_location='http://' + server + '/api', |
api_key=api_key) |
api_key=api_key) |
couch = couchdb.Server('http://127.0.0.1:5984/') |
couch = couchdb.Server('http://127.0.0.1:5984/') |
#couch = couchdb.Server('http://192.168.1.113:5984/') |
#couch = couchdb.Server('http://192.168.1.113:5984/') |
|
|
# https://github.com/okfn/ckanext-importlib |
# https://github.com/okfn/ckanext-importlib |
def munge(name): |
def munge(name): |
# convert spaces to underscores |
# convert spaces to underscores |
name = re.sub(' ', '_', name).lower() |
name = re.sub(' ', '_', name).lower() |
# convert symbols to dashes |
# convert symbols to dashes |
name = re.sub('[:]', '_-', name).lower() |
name = re.sub('[:]', '_-', name).lower() |
name = re.sub('[/]', '-', name).lower() |
name = re.sub('[/]', '-', name).lower() |
# take out not-allowed characters |
# take out not-allowed characters |
name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() |
name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() |
# remove double underscores |
# remove double underscores |
name = re.sub('__', '_', name).lower() |
name = re.sub('__', '_', name).lower() |
return name |
return name |
|
|
|
|
def name_munge(input_name): |
def name_munge(input_name): |
return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) |
return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) |
|
|
|
|
docsdb = couch['disclosr-documents'] |
docsdb = couch['disclosr-documents'] |
|
|
if __name__ == "__main__": |
if __name__ == "__main__": |
groups = {} |
groups = {} |
for doc in docsdb.view('app/datasetGroups'): |
for doc in docsdb.view('app/datasetGroups'): |
group_name = doc.key |
group_name = doc.key |
if group_name != "Not specified": |
if group_name != "Not specified": |
pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', |
pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', |
doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); |
doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); |
if group_name in groups.keys(): |
if group_name in groups.keys(): |
groups[group_name] = list(set(groups[group_name] + [pkg_name])) |
groups[group_name] = list(set(groups[group_name] + [pkg_name])) |
else: |
else: |
groups[group_name] = [pkg_name] |
groups[group_name] = [pkg_name] |
|
|
# add dataset to group(s) |
# add dataset to group(s) |
for group_name in groups.keys(): |
for group_name in groups.keys(): |
if group_name != "Not specified": |
if group_name != "Not specified": |
group_url = name_munge(group_name[:100]) |
group_url = name_munge(group_name[:100]) |
print group_name |
print group_name |
print groups[group_name] |
print groups[group_name] |
try: |
try: |
# Update the group details |
# Update the group details |
group_entity = ckan.group_entity_get(group_url) |
group_entity = ckan.group_entity_get(group_url) |
print "group "+group_name+" exists" |
print "group "+group_name+" exists" |
if 'packages' in group_entity.keys(): |
if 'packages' in group_entity.keys(): |
group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) |
group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) |
else: |
else: |
group_entity['packages'] = groups[group_name] |
group_entity['packages'] = groups[group_name] |
ckan.group_entity_put(group_entity) |
ckan.group_entity_put(group_entity) |
except CkanApiError, e: |
except CkanApiError, e: |
if ckan.last_status == 404: |
if ckan.last_status == 404: |
print "group "+group_name+" does not exist, creating" |
print "group "+group_name+" does not exist, creating" |
group_entity = { |
group_entity = { |
'name': group_url, |
'name': group_url, |
'title': group_name, |
'title': group_name, |
'description': group_name, |
'description': group_name, |
'packages': groups[group_name] |
'packages': groups[group_name] |
} |
} |
#print group_entity |
#print group_entity |
ckan.group_register_post(group_entity) |
ckan.group_register_post(group_entity) |
elif ckan.last_status == 409: |
elif ckan.last_status == 409: |
print "group already exists" |
print "group already exists" |
else: |
else: |
raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( |
raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( |
ckan.last_status, pkg_name, e.args)) |
ckan.last_status, pkg_name, e.args)) |
|
|