import ckanclient |
import ckanclient |
import couchdb |
import couchdb |
from ckanclient import CkanApiError |
from ckanclient import CkanApiError |
import re |
import re |
|
|
|
|
class LoaderError(Exception): |
class LoaderError(Exception): |
pass |
pass |
|
|
# Instantiate the CKAN client. |
# Instantiate the CKAN client. |
#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') |
#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') |
api_key = 'b3ab75e4-afbb-465b-a09d-8171c8c69a7a' |
api_key = 'b3ab75e4-afbb-465b-a09d-8171c8c69a7a' |
ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', |
ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', |
api_key=api_key) |
api_key=api_key) |
couch = couchdb.Server('http://127.0.0.1:5984/') |
couch = couchdb.Server('http://127.0.0.1:5984/') |
#couch = couchdb.Server('http://192.168.1.113:5984/') |
#couch = couchdb.Server('http://192.168.1.113:5984/') |
|
|
# https://github.com/okfn/ckanext-importlib |
# https://github.com/okfn/ckanext-importlib |
def munge(name): |
def munge(name): |
# convert spaces to underscores |
# convert spaces to underscores |
name = re.sub(' ', '_', name).lower() |
name = re.sub(' ', '_', name).lower() |
# convert symbols to dashes |
# convert symbols to dashes |
name = re.sub('[:]', '_-', name).lower() |
name = re.sub('[:]', '_-', name).lower() |
name = re.sub('[/]', '-', name).lower() |
name = re.sub('[/]', '-', name).lower() |
# take out not-allowed characters |
# take out not-allowed characters |
name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() |
name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() |
# remove double underscores |
# remove double underscores |
name = re.sub('__', '_', name).lower() |
name = re.sub('__', '_', name).lower() |
return name |
return name |
|
|
|
|
def name_munge(input_name): |
def name_munge(input_name): |
return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) |
return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) |
|
|
|
|
docsdb = couch['disclosr-documents'] |
docsdb = couch['disclosr-documents'] |
|
|
if __name__ == "__main__": |
if __name__ == "__main__": |
groups = {} |
groups = {} |
for doc in docsdb.view('app/datasetGroups'): |
for doc in docsdb.view('app/datasetGroups'): |
group_name = doc.key |
group_name = doc.key |
pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', |
if group_name != "Not specified": |
doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); |
pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', |
if group_name in groups.keys(): |
doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); |
groups[group_name] = list(set(groups[group_name] + [pkg_name])) |
if group_name in groups.keys(): |
else: |
groups[group_name] = list(set(groups[group_name] + [pkg_name])) |
groups[group_name] = [pkg_name] |
else: |
|
groups[group_name] = [pkg_name] |
|
|
# add dataset to group(s) |
# add dataset to group(s) |
for group_name in groups.keys(): |
for group_name in groups.keys(): |
group_url = name_munge(group_name[:100]) |
if group_name != "Not specified": |
print group_name |
group_url = name_munge(group_name[:100]) |
print groups[group_name] |
print group_name |
try: |
print groups[group_name] |
# Update the group details |
try: |
group_entity = ckan.group_entity_get(group_url) |
# Update the group details |
print "group "+group_name+" exists" |
group_entity = ckan.group_entity_get(group_url) |
if 'packages' in group_entity.keys(): |
print "group "+group_name+" exists" |
group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) |
if 'packages' in group_entity.keys(): |
else: |
group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) |
group_entity['packages'] = groups[group_name] |
else: |
ckan.group_entity_put(group_entity) |
group_entity['packages'] = groups[group_name] |
except CkanApiError, e: |
ckan.group_entity_put(group_entity) |
if ckan.last_status == 404: |
except CkanApiError, e: |
print "group "+group_name+" does not exist, creating" |
if ckan.last_status == 404: |
group_entity = { |
print "group "+group_name+" does not exist, creating" |
'name': group_url, |
group_entity = { |
'title': group_name, |
'name': group_url, |
'description': group_name, |
'title': group_name, |
'packages': groups[group_name] |
'description': group_name, |
} |
'packages': groups[group_name] |
#print group_entity |
} |
ckan.group_register_post(group_entity) |
#print group_entity |
elif ckan.last_status == 409: |
ckan.group_register_post(group_entity) |
print "group already exists" |
elif ckan.last_status == 409: |
else: |
print "group already exists" |
raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( |
else: |
ckan.last_status, pkg_name, e.args)) |
raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( |
|
ckan.last_status, pkg_name, e.args)) |
|
|