1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | import ckanclient import couchdb from ckanclient import CkanApiError import re class LoaderError(Exception): pass # Instantiate the CKAN client. ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b') # (use your own api_key from http://thedatahub.org/user/me ) # http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/ SYMBOLS = { 'customary': ('B', 'KB', 'MB', 'GB', 'T', 'P', 'E', 'Z', 'Y'), 'customary_ext': ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa', 'zetta', 'iotta'), 'iec': ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'), 'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi', 'zebi', 'yobi'), } def human2bytes(s): """ Attempts to guess the string format based on default symbols set and return the corresponding bytes as an integer. When unable to recognize the format ValueError is raised. >>> human2bytes('0 B') 0 >>> human2bytes('1 K') 1024 >>> human2bytes('1 M') 1048576 >>> human2bytes('1 Gi') 1073741824 >>> human2bytes('1 tera') 1099511627776 >>> human2bytes('0.5kilo') 512 >>> human2bytes('0.1 byte') 0 >>> human2bytes('1 k') # k is an alias for K 1024 >>> human2bytes('12 foo') Traceback (most recent call last): ... ValueError: can't interpret '12 foo' """ init = s num = "" while s and s[0:1].isdigit() or s[0:1] == '.': num += s[0] s = s[1:] num = float(num) letter = s.strip() for name, sset in SYMBOLS.items(): if letter in sset: break else: if letter == 'k': # treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs sset = SYMBOLS['customary'] letter = letter.upper() else: raise ValueError("can't interpret %r" % init) prefix = {sset[0]: 1} for i, s in enumerate(sset[1:]): prefix[s] = 1 << (i + 1) * 10 return int(num * prefix[letter]) # https://github.com/okfn/ckanext-importlib def munge(name): # convert spaces to underscores name = re.sub(' ', '_', name).lower() # convert symbols to dashes name = re.sub('[:]', '_-', name).lower() name = re.sub('[/]', '-', name).lower() # take out not-allowed characters name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() # remove double underscores name = re.sub('__', '_', name).lower() return name def name_munge(input_name): return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and')) #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and') couch = couchdb.Server('http://127.0.0.1:5984/') docsdb = couch['disclosr-documents'] if __name__ == "__main__": for doc in docsdb.view('app/datasets'): print doc.id if doc.value['url'] != "http://data.gov.au/data/": # Collect the package metadata. pkg_name = name_munge(doc.value['metadata']['DCTERMS.Title'][:100]) tags = doc.value['metadata']["Keywords / Tags"] if not hasattr(tags, '__iter__'): tags = [tags] [re.sub('[^a-zA-Z0-9-_]', '', tag).lower() for tag in tags] package_entity = { 'name': pkg_name, 'title': doc.value['metadata']['DCTERMS.Title'], 'url': doc.value['metadata']['DCTERMS.Source.URI'], 'tags': tags, 'author': doc.value['metadata']["DCTERMS.Creator"], 'maintainer': doc.value['metadata']["DCTERMS.Creator"], 'licence_id': doc.value['metadata']['DCTERMS.License'], #todo licence id mapping 'notes': doc.value['metadata']['Description'], } try: #print doc.id ckan.package_register_post(package_entity) except CkanApiError, e: if ckan.last_status == 409: print "already exists" else: raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args)) print package_entity #todo add to organisation (author/creator/maintainer) #if 'data.gov.au Category' in doc.value['metadata'].keys(): #todo add to group if 'Download' in doc.value['metadata'].keys(): try: pkg = ckan.package_entity_get(pkg_name) resources = pkg.get('resources', []) if len(resources) < len(doc.value['metadata']['Download']): for resource in doc.value['metadata']['Download']: #print resource # http://docs.ckan.org/en/ckan-1.7/domain-model-resource.html # (KML/KMZ) / (Shapefile) /(Other) format = "plain" if resource['format'] == '(XML)': format = 'xml' if resource['format'] == '(CSV/XLS)': format = 'csv' name = resource['href'] if 'name' in resource.keys(): name = resource['name'] ckan.add_package_resource(pkg_name, resource['href'], name=name, resource_type='data', format=format, size=human2bytes(resource['size'].replace(',', ''))) else: print "resources already exist" except CkanApiError, e: if ckan.last_status == 404: print "parent dataset does not exist" else: raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % ( ckan.last_status, pkg_name, e.args)) |