1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 | import ckanclient import couchdb from ckanclient import CkanApiError import re import html2text # aaronsw :( import ckanapi # https://github.com/open-data/ckanapi class LoaderError(Exception): pass # Instantiate the CKAN client. api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc' server = 'data.disclosurelo.gs' ckan = ckanclient.CkanClient(base_location='http://'+server+'api', api_key=api_key) ckandirect = ckanapi.RemoteCKAN('http://'+server, api_key=api_key) couch = couchdb.Server('http://127.0.0.1:5984/') #couch = couchdb.Server('http://192.168.1.113:5984/') import urllib import urlparse def url_fix(s, charset='utf-8'): """Sometimes you get an URL by a user that just isn't a real URL because it contains unsafe characters like ' ' and so on. This function can fix some of the problems in a similar way browsers handle data entered by the user: >>> url_fix(u'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)') 'http://de.wikipedia.org/wiki/Elf%20%28Begriffskl%C3%A4rung%29' :param charset: The target charset for the URL if the url was given as unicode string. """ if isinstance(s, unicode): s = s.encode(charset, 'ignore') if not urlparse.urlparse(s).scheme: s = "http://"+s scheme, netloc, path, qs, anchor = urlparse.urlsplit(s) path = urllib.quote(path, '/%') qs = urllib.quote_plus(qs, ':&=') return urlparse.urlunsplit((scheme, netloc, path, qs, anchor)) # http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/ SYMBOLS = { 'customary': ('B', 'KB', 'MB', 'GB', 'T', 'P', 'E', 'Z', 'Y'), 'customary_ext': ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa', 'zetta', 'iotta'), 'iec': ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'), 'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi', 'zebi', 'yobi'), } def human2bytes(s): """ Attempts to guess the string format based on default symbols set and return the corresponding bytes as an integer. When unable to recognize the format ValueError is raised. >>> human2bytes('0 B') 0 >>> human2bytes('1 K') 1024 >>> human2bytes('1 M') 1048576 >>> human2bytes('1 Gi') 1073741824 >>> human2bytes('1 tera') 1099511627776 >>> human2bytes('0.5kilo') 512 >>> human2bytes('0.1 byte') 0 >>> human2bytes('1 k') # k is an alias for K 1024 >>> human2bytes('12 foo') Traceback (most recent call last): ... ValueError: can't interpret '12 foo' """ if s == None: return 0 s = s.replace(',', '') init = s num = "" while s and s[0:1].isdigit() or s[0:1] == '.': num += s[0] s = s[1:] num = float(num) letter = s.strip() for name, sset in SYMBOLS.items(): if letter in sset: break else: if letter == 'k': # treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs sset = SYMBOLS['customary'] letter = letter.upper() else: raise ValueError("can't interpret %r" % init) prefix = {sset[0]: 1} for i, s in enumerate(sset[1:]): prefix[s] = 1 << (i + 1) * 10 return int(num * prefix[letter]) # https://github.com/okfn/ckanext-importlib def munge(name): # convert spaces to underscores name = re.sub(' ', '_', name).lower() # convert symbols to dashes name = re.sub('[:]', '_-', name).lower() name = re.sub('[/]', '-', name).lower() # take out not-allowed characters name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() # remove double underscores name = re.sub('__', '_', name).lower() return name def name_munge(input_name): return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) def get_licence_id(licencename): map = { "Creative Commons - Attribution-Share Alike 2.0 Australia (CC-SA)\nThe downloadable version of the database is licensed under CC-BY-SA Creative Commons Attribution Share Alike and contains only the database fields that are released under that license. These fields are object title, object number, object description as well as temporal, spatial and dimension details. It also contains a persistent URL for each record.": 'cc-by-sa', "CreativeCommonsAttributionNonCommercial30AustraliaCCBYNC30": 'cc-nc', 'Otherpleasespecify': 'notspecified', '': 'notspecified', "Publicly available data": 'notspecified', "CreativeCommonsAttributionNoDerivativeWorks30AustraliaCCBYND30": "other-closed", "CreativeCommonsAttributionNonCommercialNoDerivs30AustraliaCCBYNCND30": "other-closed", 'CreativeCommonsAttribution30AustraliaCCBY30': 'cc-by', "Creative Commons - Attribution 2.5 Australia (CC-BY)": 'cc-by', 'CreativeCommonsAttributionCCBY25': 'cc-by', "PublicDomain": 'other-pd', } if licencename not in map.keys(): raise Exception(licencename + " not found"); return map[licencename]; docsdb = couch['disclosr-documents'] if __name__ == "__main__": orgs_list = [] orgs_ids = {} for doc in docsdb.view('app/datasets'): print " --- " print doc.id if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld": # Collect the package metadata. pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_', doc.value['url'].replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]); print pkg_name #add to or create organization using direct API agency = doc.value['metadata']["Agency"] if agency == "APS": agency = "Australian Public Service Commission" if agency == "Shared Services, Treasury Directorate": agency = "Shared Services Procurement, Treasury Directorate" if agency == "Treasury - Shared Services": agency = "Shared Services Procurement, Treasury Directorate" if agency == "Territory and Municipal Services (TAMS)": agency = "Territory and Municipal Services Directorate" if agency == "State Library of NSW": agency = "State Library of New South Wales" org_name = name_munge(agency[:100]) if org_name not in orgs_list: orgs_list = ckandirect.action.organization_list()['result'] #print orgs_list if org_name not in orgs_list: try: print "org not found, creating " + org_name ckandirect.action.organization_create(name=org_name, title=agency, description=agency) orgs_list.append(org_name) except ckanapi.ValidationError, e: print e raise LoaderError('Unexpected status') else: print "org found, adding dataset to " + org_name # cache org names -> id mapping if org_name not in orgs_ids: org = ckandirect.action.organization_show(id=org_name) orgs_ids[org_name] = org["result"]["id"] org_id = orgs_ids[org_name] print "org id is "+org_id tags = [] creator = doc.value['metadata']["DCTERMS.Creator"] if doc.value['agencyID'] == "AGIMO": if len(doc.value['metadata']["Keywords / Tags"]) > 0: if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'): tags = tags + doc.value['metadata']["Keywords / Tags"] else: tags = tags + [doc.value['metadata']["Keywords / Tags"]] tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag] #print tags extras = [] for extra_key in doc.value['metadata'].keys(): if extra_key not in ["Description","Content-Language","DCTERMS.Description", "Keywords / Tags" ,"data.gov.au Category", "Download", "Permalink","DCTERMS.Identifier"]: if doc.value['metadata'][extra_key] != None and doc.value['metadata'][extra_key] != "": extras.append([extra_key, doc.value['metadata'][extra_key]]) package_entity = { 'name': pkg_name, 'title': doc.value['metadata']['DCTERMS.Title'], 'url': doc.value['metadata']['DCTERMS.Source.URI'], 'tags': tags, #tags are mandatory? 'author': creator, 'maintainer': creator, 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']), 'notes': html2text.html2text(doc.value['metadata']['Description']), 'owner_org': org_id, 'extras': extras } try: #print package_entity ckan.package_register_post(package_entity) except CkanApiError, e: if ckan.last_message == "{\"name\": [\"That URL is already in use.\"]}": print "package already exists" else: print ckan.last_message raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args)) pkg = ckan.package_entity_get(pkg_name) # add resources (downloadable data files) if 'Download' in doc.value['metadata'].keys(): try: resources = pkg.get('resources', []) if len(resources) < len(doc.value['metadata']['Download']): for resource in doc.value['metadata']['Download']: # http://docs.ckan.org/en/ckan-1.7/domain-model-resource.html # (KML/KMZ) / (Shapefile) /(Other) format = "plain" if resource['format'] == '(XML)': format = 'xml' if resource['format'] == '(CSV/XLS)': format = 'csv' if resource['format'] == '(Shapefile)': format = 'shp' if resource['format'] == '(KML/KMZ)': format = 'kml' name = resource['href'] if 'name' in resource.keys(): name = resource['name'] print resource ckan.add_package_resource(pkg_name, url_fix(resource['href']), name=name, resource_type='data', format=format, size=human2bytes(resource.get('size','0B'))) else: print "resources already exist" except CkanApiError, e: if ckan.last_status == 404: print "parent dataset does not exist" else: raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args)) |