--- a/documents/datagov-export.py +++ b/documents/datagov-export.py @@ -12,8 +12,8 @@ #ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', api_key='482a9dd2-a976-4adf-ac77-d71d92a98a52') -#couch = couchdb.Server('http://127.0.0.1:5984/') -couch = couchdb.Server('http://192.168.1.113:5984/') +couch = couchdb.Server('http://127.0.0.1:5984/') +#couch = couchdb.Server('http://192.168.1.113:5984/') # http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/ SYMBOLS = { @@ -91,6 +91,7 @@ def name_munge(input_name): return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) + #[:100] #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and') @@ -117,32 +118,35 @@ if __name__ == "__main__": for doc in docsdb.view('app/datasets'): print doc.id - if doc.value['url'] != "http://data.gov.au/data/": + if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld": # Collect the package metadata. - pkg_name = name_munge(doc.value['metadata']['DCTERMS.Title'][:100]) + pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/',''); tags = [] - if len(doc.value['metadata']["Keywords / Tags"]) > 0: - if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'): - tags = tags + doc.value['metadata']["Keywords / Tags"] - else: - tags = tags + [doc.value['metadata']["Keywords / Tags"]] - if 'data.gov.au Category' in doc.value['metadata'].keys() and len(doc.value['metadata']['data.gov.au Category']) > 0: - if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'): - tags = tags + doc.value['metadata']['data.gov.au Category'] - else: - tags = tags + [doc.value['metadata']['data.gov.au Category']] - tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag] - print tags - package_entity = { - 'name': pkg_name, - 'title': doc.value['metadata']['DCTERMS.Title'], - 'url': doc.value['metadata']['DCTERMS.Source.URI'], - 'tags': tags, #tags are mandatory? - 'author': doc.value['metadata']["DCTERMS.Creator"], - 'maintainer': doc.value['metadata']["DCTERMS.Creator"], - 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']), - 'notes': html2text.html2text(doc.value['metadata']['Description']), - } + if doc.value['agencyID'] == "AGIMO": + if len(doc.value['metadata']["Keywords / Tags"]) > 0: + if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'): + tags = tags + doc.value['metadata']["Keywords / Tags"] + else: + tags = tags + [doc.value['metadata']["Keywords / Tags"]] + if 'data.gov.au Category' in doc.value['metadata'].keys() and len(doc.value['metadata']['data.gov.au Category']) > 0: + if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'): + tags = tags + doc.value['metadata']['data.gov.au Category'] + else: + tags = tags + [doc.value['metadata']['data.gov.au Category']] + tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag] + print tags + package_entity = { + 'name': pkg_name, + 'title': doc.value['metadata']['DCTERMS.Title'], + 'url': doc.value['metadata']['DCTERMS.Source.URI'], + 'tags': tags, #tags are mandatory? + 'author': doc.value['metadata']["DCTERMS.Creator"], + 'maintainer': doc.value['metadata']["DCTERMS.Creator"], + 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']), + 'notes': html2text.html2text(doc.value['metadata']['Description']), + } + if doc.value['agencyID'] == "qld": + package_entity = doc.value['metadata'] try: print package_entity @@ -182,6 +186,8 @@ } print group_entity ckan.group_register_post(group_entity) + elif ckan.last_status == 409: + print "group already exists" else: raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args))