--- a/documents/datagov-export.py
+++ b/documents/datagov-export.py
@@ -10,8 +10,7 @@
     pass
 
 # Instantiate the CKAN client.
-#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api',    api_key='b47b24cd-591d-40c1-8677-d73101d56d1b')
-api_key = 'b3ab75e4-afbb-465b-a09d-8171c8c69a7a'
+api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc'
 ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api',
                              api_key=api_key)
 ckandirect = ckanapi.RemoteCKAN('http://data.disclosurelo.gs', api_key=api_key)
@@ -57,6 +56,9 @@
           ...
       ValueError: can't interpret '12 foo'
     """
+    if s == None:
+	return 0
+    s = s.replace(',', '')
     init = s
     num = ""
     while s and s[0:1].isdigit() or s[0:1] == '.':
@@ -92,12 +94,9 @@
     name = re.sub('__', '_', name).lower()
     return name
 
-#todo "{'name': [u'Url must be purely lowercase alphanumeric (ascii) characters and these symbols: -_']}"
-# http://data.gov.au/dataset/australian-domestic-regional-and-international-airline-activity-%E2%80%93-time-series/
+
 def name_munge(input_name):
     return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
-    #[:100]
-    #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and')
 
 
 def get_licence_id(licencename):
@@ -123,7 +122,9 @@
 
 if __name__ == "__main__":
     orgs_list = []
+    orgs_ids = {}
     for doc in docsdb.view('app/datasets'):
+        print "   ---   "
         print doc.id
 
         if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
@@ -150,8 +151,12 @@
                 else:
                     print "org found, adding dataset to " + org_name
 
-            org = ckandirect.action.organization_show(id=org_name)
-            # todo cache org names -> id mapping
+            # cache org names -> id mapping
+            if org_name not in orgs_ids:
+                org = ckandirect.action.organization_show(id=org_name)
+                orgs_ids[org_name] = org["result"]["id"]
+            org_id = orgs_ids[org_name]
+            print "org id is "+org_id
             tags = []
             if doc.value['agencyID'] == "AGIMO":
                 if len(doc.value['metadata']["Keywords / Tags"]) > 0:
@@ -171,7 +176,7 @@
                     'maintainer': doc.value['metadata']["DCTERMS.Creator"],
                     'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
                     'notes': html2text.html2text(doc.value['metadata']['Description']),
-                    'owner_org': org["result"]["id"]
+                    'owner_org': org_id
                     #todo add missing key values like jurasdiction
                 }
             if doc.value['agencyID'] == "qld":
@@ -189,45 +194,6 @@
                         ckan.last_status, pkg_name, e.args))
             pkg = ckan.package_entity_get(pkg_name)
 
-            # add dataset to group(s)
-            groups = []
-            if 'data.gov.au Category' in doc.value['metadata'].keys() and len(
-                    doc.value['metadata']['data.gov.au Category']) > 0:
-                if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'):
-                    groups = groups + doc.value['metadata']['data.gov.au Category']
-                else:
-                    groups = groups + [doc.value['metadata']['data.gov.au Category']]
-
-            for group_name in groups:
-                group_url = name_munge(group_name[:100])
-                try:
-                    print ckan.group_entity_get(group_url)
-
-                    # Update the group details
-                    group_entity = ckan.last_message
-                    print "group "+group_name+" exists"
-                    if 'packages' in group_entity.keys():
-                        group_entity['packages'] = list(set(group_entity['packages'] + [pkg_name]))
-                    else:
-                        group_entity['packages'] = [pkg_name]
-                    ckan.group_entity_put(group_entity)
-                except CkanApiError, e:
-                    if ckan.last_status == 404:
-                        print "group "+group_name+" does not exist, creating"
-                        group_entity = {
-                            'name': group_url,
-                            'title': group_name,
-                            'description': group_name,
-                            'packages': [pkg_name]
-                        }
-                        #print group_entity
-                        ckan.group_register_post(group_entity)
-                    elif ckan.last_status == 409:
-                        print "group already exists"
-                    else:
-                        raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % (
-                            ckan.last_status, pkg_name, e.args))
-
             # add resources (downloadable data files)
             if 'Download' in doc.value['metadata'].keys():
                 try:
@@ -243,13 +209,17 @@
                                 format = 'xml'
                             if resource['format'] == '(CSV/XLS)':
                                 format = 'csv'
+                            if resource['format'] == '(Shapefile)':
+                                format = 'shp'
+                            if resource['format'] == '(KML/KMZ)':
+                                format = 'kml'
                             name = resource['href']
                             if 'name' in resource.keys():
                                 name = resource['name']
                             print resource
                             ckan.add_package_resource(pkg_name, resource['href'], name=name, resource_type='data',
                                                       format=format,
-                                                      size=human2bytes(resource['size'].replace(',', '')))
+                                                      size=human2bytes(resource.get('size','0B')))
                     else:
                         print "resources already exist"
                 except CkanApiError, e: