--- a/documents/datagov-export.py +++ b/documents/datagov-export.py @@ -13,22 +13,38 @@ pass import tempfile -def add_package_resource_cachedurl(ckan, package_name, url, name, format, size, **kwargs): +def add_package_resource_cachedurl(ckan, package_name, url, name, format, license_id, size,**kwargs): + excluded_apis = "recent-earthquakes,sentinel-hotspots,abc-local-stations,action-bus-service-gtfs-feed-act,current-and-future-road-reports-traffic-restrictions-in-south-australia,current-and-future-road-reports-traffic-restrictions-in-south-australia,current-and-future-road-reports-traffic-restrictions-in-south-australia,current-and-future-road-reports-traffic-restrictions-in-south-australia,current-and-future-road-reports-traffic-restrictions-in-south-australia,current-and-future-road-reports-traffic-restrictions-in-south-australia,journey-planner-data-act,live-traffic-cameras-nsw,precis-forecast-national,precis-forecast-national,precis-forecast-new-south-wales,precis-forecast-new-south-wales,precis-forecast-northern-territory,precis-forecast-northern-territory,precis-forecast-queensland,precis-forecast-queensland,precis-forecast-south-australia,precis-forecast-south-australia,precis-forecast-tasmania,precis-forecast-tasmania,precis-forecast-victoria,precis-forecast-victoria,precis-forecast-western-australia,precis-forecast-western-australia,register-of-penalty-notices-nsw,sentinel-hotspots,trove-people-and-organisations-data,weather-data-services-radar,abc-local-stations,act-emergency-services-agency-esa-28093-current-incidents,act-emergency-services-agency-esa-news-alerts,act-government-news-and-events,act-government-summaries-of-cabinet-outcomes,act-magistrates-court-judgements,act-supreme-court-judgements,act-supreme-court-sentences,actpla-latest-news,all-vacant-act-government-jobs,community-engagement-current-engagements,community-engagement-news,edd-media-releases,edd-news-and-events,freedom-of-information-foi-summaries,libraries-act-announcements,nsw-rural-fire-service-current-incidents,nsw-rural-fire-service-major-updates,precis-forecast-new-south-wales,precis-forecast-south-australia,precis-forecast-tasmania,precis-forecast-victoria,sentinel-hotspots,south-australian-road-crash-statistics,trove-people-and-organisations-data,weather-warnings-for-new-south-wales-australian-capital-territory,weather-warnings-for-northern-territory,weather-warnings-for-queensland,weather-warnings-for-south-australia,weather-warnings-for-tasmania,weather-warnings-for-victoria,weather-warnings-for-western-australia".split(",") + if "xls" in url: + format = "xls" + if "pdf" in url: + format = "pdf" + if "xlsx" in url: + format = "xlsx" (returned_url, mime_type, content) = scrape.fetchURL(scrape.docsdb, url, "dataset_resource", "AGIMO", False) - if content != None: - tf = tempfile.NamedTemporaryFile(delete=False) + if mime_type in ["application/vnd.ms-excel","application/msexcel","application/x-msexcel","application/x-ms-excel","application/x-excel","application/x-dos_ms_excel","application/xls","application/x-xls"]: + format = "xls" + if mime_type in ["application/xlsx","application/x-xlsx","application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"]: + format = "xlsx" + + #put file extensions on for windows users downloading files + suffix = name.encode("ascii","ignore").replace("/","") + if len(suffix) < 5 or (suffix[-4] != "." and suffix[-5] != "."): + suffix = suffix + "." + format + if content != None and package_name not in excluded_apis: + tf = tempfile.NamedTemporaryFile(suffix=suffix) tfName = os.path.abspath(tf.name) print tfName tf.seek(0) tf.write(content) tf.flush() - ckan.add_package_resource (package_name, tfName, name=name) + ckan.add_package_resource (package_name, tfName, name=name, format=format, license_id=license_id) else: print "fetch error" - ckan.add_package_resource(package_name, url, name=name, resource_type='data', + return ckan.add_package_resource(package_name, url, name=name, resource_type='data', format=format, - size=size) + size=size, mimetype=mime_type, license_id=license_id) # Instantiate the CKAN client. api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' @@ -147,15 +163,15 @@ return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) -def get_licence_id(licencename): +def get_license_id(licencename): map = { "Creative Commons - Attribution-Share Alike 2.0 Australia (CC-SA)\nThe downloadable version of the database is licensed under CC-BY-SA Creative Commons Attribution Share Alike and contains only the database fields that are released under that license. These fields are object title, object number, object description as well as temporal, spatial and dimension details. It also contains a persistent URL for each record.": 'cc-by-sa', "CreativeCommonsAttributionNonCommercial30AustraliaCCBYNC30": 'cc-nc', 'Otherpleasespecify': 'notspecified', '': 'notspecified', "Publicly available data": 'notspecified', - "CreativeCommonsAttributionNoDerivativeWorks30AustraliaCCBYND30": "other-closed", - "CreativeCommonsAttributionNonCommercialNoDerivs30AustraliaCCBYNCND30": "other-closed", + "CreativeCommonsAttributionNoDerivativeWorks30AustraliaCCBYND30": "cc-by-nd", + "CreativeCommonsAttributionNonCommercialNoDerivs30AustraliaCCBYNCND30": "cc-nc-nd", 'CreativeCommonsAttribution30AustraliaCCBY30': 'cc-by', "Creative Commons - Attribution 2.5 Australia (CC-BY)": 'cc-by', 'CreativeCommonsAttributionCCBY25': 'cc-by', @@ -185,12 +201,14 @@ pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', doc.value['url'].replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); print pkg_name - if pkg_name != "": + if pkg_name != "" : #add to or create organization using direct API agency = doc.value['metadata']["Agency"] if agency == "APS": agency = "Australian Public Service Commission" + if agency == "Department of Broadband, Communications and the Digital Ecomomy": + agency = "Department of Broadband, Communications and the Digital Economy" if agency == "Shared Services, Treasury Directorate": agency = "Shared Services Procurement, Treasury Directorate" if agency == "Treasury - Shared Services": @@ -248,8 +266,8 @@ 'tags': tags, #tags are mandatory? 'author': creator, 'maintainer': creator, - 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']), - 'notes': html2text.html2text(doc.value['metadata']['Description']), + 'license_id': get_license_id(doc.value['metadata']['DCTERMS.License']), + 'notes': html2text.html2text(doc.value['metadata']['Description']).replace('AC/a!a','-').replace('AC/a!aC/',"'").replace("AC/a!E",":").replace("A "," "), 'owner_org': org_id, 'extras': extras, 'private': (pkg_name not in goodcsvdata and pkg_name not in goodotherdata) @@ -292,9 +310,8 @@ name = resource['name'] print resource add_package_resource_cachedurl(ckan, pkg_name, url_fix(resource['href']), name, - format, - human2bytes(resource.get('size', '0B')), - resource_type='data') + format, get_license_id(doc.value['metadata']['DCTERMS.License']), + human2bytes(resource.get('size', '0B'))) else: print "resources already exist" except CkanApiError, e: