1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | import ckanclient import couchdb from ckanclient import CkanApiError import re class LoaderError(Exception): pass # Instantiate the CKAN client. #ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' server = 'data.disclosurelo.gs' api_key = 'c30eb6f5-0f90-47e0-bf05-9b1b4e3a461a' server = 'ckan.data.gov.au' ckan = ckanclient.CkanClient(base_location='http://' + server + '/api', api_key=api_key) couch = couchdb.Server('http://127.0.0.1:5984/') #couch = couchdb.Server('http://192.168.1.113:5984/') # https://github.com/okfn/ckanext-importlib def munge(name): # convert spaces to underscores name = re.sub(' ', '_', name).lower() # convert symbols to dashes name = re.sub('[:]', '_-', name).lower() name = re.sub('[/]', '-', name).lower() # take out not-allowed characters name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() # remove double underscores name = re.sub('__', '_', name).lower() return name def name_munge(input_name): return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) docsdb = couch['disclosr-documents'] if __name__ == "__main__": groups = {} for doc in docsdb.view('app/datasetGroups'): group_name = doc.key if group_name != "Not specified": pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); if group_name in groups.keys(): groups[group_name] = list(set(groups[group_name] + [pkg_name])) else: groups[group_name] = [pkg_name] # add dataset to group(s) for group_name in groups.keys(): if group_name != "Not specified": group_url = name_munge(group_name[:100]) print group_name print groups[group_name] try: # Update the group details group_entity = ckan.group_entity_get(group_url) print "group "+group_name+" exists" if 'packages' in group_entity.keys(): group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name])) else: group_entity['packages'] = groups[group_name] ckan.group_entity_put(group_entity) except CkanApiError, e: if ckan.last_status == 404: print "group "+group_name+" does not exist, creating" group_entity = { 'name': group_url, 'title': group_name, 'description': group_name, 'packages': groups[group_name] } #print group_entity ckan.group_register_post(group_entity) elif ckan.last_status == 409: print "group already exists" else: raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args)) |