--- a/documents/datagov-export.py +++ b/documents/datagov-export.py @@ -10,8 +10,7 @@ pass # Instantiate the CKAN client. -#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') -api_key = 'b3ab75e4-afbb-465b-a09d-8171c8c69a7a' +api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api', api_key=api_key) ckandirect = ckanapi.RemoteCKAN('http://data.disclosurelo.gs', api_key=api_key) @@ -57,6 +56,9 @@ ... ValueError: can't interpret '12 foo' """ + if s == None: + return 0 + s = s.replace(',', '') init = s num = "" while s and s[0:1].isdigit() or s[0:1] == '.': @@ -177,6 +179,12 @@ tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag] #print tags + extras = [] + + for extra_key in doc.value['metadata'].keys(): + if extra_key != "Keywords / Tags" and extra_key != "data.gov.au Category" and extra_key != "Download" : + extras.append({'key':extra_key, 'value':doc.value['metadata'][extra_key]}) + package_entity = { 'name': pkg_name, 'title': doc.value['metadata']['DCTERMS.Title'], @@ -186,8 +194,8 @@ 'maintainer': creator, 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']), 'notes': html2text.html2text(doc.value['metadata']['Description']), - 'owner_org': org_id - #todo add missing key values like jurasdiction + 'owner_org': org_id, + 'extras': extras } @@ -219,13 +227,17 @@ format = 'xml' if resource['format'] == '(CSV/XLS)': format = 'csv' + if resource['format'] == '(Shapefile)': + format = 'shp' + if resource['format'] == '(KML/KMZ)': + format = 'kml' name = resource['href'] if 'name' in resource.keys(): name = resource['name'] print resource ckan.add_package_resource(pkg_name, resource['href'], name=name, resource_type='data', format=format, - size=human2bytes(resource['size'].replace(',', ''))) + size=human2bytes(resource.get('size','0B'))) else: print "resources already exist" except CkanApiError, e: