| import ckanclient | |
| import couchdb | |
| from ckanclient import CkanApiError | |
| import re | |
| class LoaderError(Exception): | |
| pass | |
| # Instantiate the CKAN client. | |
| #ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') | |
| api_key = 'b3ab75e4-afbb-465b-a09d-8171c8c69a7a' | |
| ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', | |
| api_key=api_key) | |
| couch = couchdb.Server('http://127.0.0.1:5984/') | |
| #couch = couchdb.Server('http://192.168.1.113:5984/') | |
| # https://github.com/okfn/ckanext-importlib | |
| def munge(name): | |
| # convert spaces to underscores | |
| name = re.sub(' ', '_', name).lower() | |
| # convert symbols to dashes | |
| name = re.sub('[:]', '_-', name).lower() | |
| name = re.sub('[/]', '-', name).lower() | |
| # take out not-allowed characters | |
| name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() | |
| # remove double underscores | |
| name = re.sub('__', '_', name).lower() | |
| return name | |
| def name_munge(input_name): | |
| return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) | |
| docsdb = couch['disclosr-documents'] | |
| if __name__ == "__main__": | |
| groups = {} | |
| for doc in docsdb.view('app/datasetGroups'): | |
| group_name = doc.key | |
| pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', | |
| doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); | |
| if group_name in groups.keys(): | |
| groups[group_name] = list(set(groups[group_name] + [pkg_name])) | |
| else: | |
| groups[group_name] = [pkg_name] | |
| # add dataset to group(s) | |
| for group_name in groups.keys(): | |
| group_url = name_munge(group_name[:100]) | |
| print group_name | |
| print groups[group_name] | |
| try: | |
| # Update the group details | |
| group_entity = ckan.group_entity_get(group_url) | |
| print "group "+group_name+" exists" | |
| if 'packages' in group_entity.keys(): | |
| group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) | |
| else: | |
| group_entity['packages'] = groups[group_name] | |
| ckan.group_entity_put(group_entity) | |
| except CkanApiError, e: | |
| if ckan.last_status == 404: | |
| print "group "+group_name+" does not exist, creating" | |
| group_entity = { | |
| 'name': group_url, | |
| 'title': group_name, | |
| 'description': group_name, | |
| 'packages': groups[group_name] | |
| } | |
| #print group_entity | |
| ckan.group_register_post(group_entity) | |
| elif ckan.last_status == 409: | |
| print "group already exists" | |
| else: | |
| raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( | |
| ckan.last_status, pkg_name, e.args)) | |