From: Alex Sadleir <maxious@lambdacomplex.org>
Date: Sun, 14 Apr 2013 09:42:53 +0000
Subject: cache org ids while importing datagov
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=f5133c3240e293b5bbc84ff2cb6a00d1ea8bd900
---
cache org ids while importing datagov


Former-commit-id: 3bb1449efe6725785ffb74bfe250e27913e47885
---


--- a/documents/datagov-export.py
+++ b/documents/datagov-export.py
@@ -92,12 +92,9 @@
     name = re.sub('__', '_', name).lower()
     return name
 
-#todo "{'name': [u'Url must be purely lowercase alphanumeric (ascii) characters and these symbols: -_']}"
-# http://data.gov.au/dataset/australian-domestic-regional-and-international-airline-activity-%E2%80%93-time-series/
+
 def name_munge(input_name):
     return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
-    #[:100]
-    #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and')
 
 
 def get_licence_id(licencename):
@@ -123,7 +120,9 @@
 
 if __name__ == "__main__":
     orgs_list = []
+    orgs_ids = {}
     for doc in docsdb.view('app/datasets'):
+        print "   ---   "
         print doc.id
 
         if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
@@ -150,8 +149,12 @@
                 else:
                     print "org found, adding dataset to " + org_name
 
-            org = ckandirect.action.organization_show(id=org_name)
-            # todo cache org names -> id mapping
+            # cache org names -> id mapping
+            if org_name not in orgs_ids:
+                org = ckandirect.action.organization_show(id=org_name)
+                orgs_ids[org_name] = org["result"]["id"]
+            org_id = orgs_ids[org_name]
+            print "org id is "+org_id
             tags = []
             if doc.value['agencyID'] == "AGIMO":
                 if len(doc.value['metadata']["Keywords / Tags"]) > 0:
@@ -171,7 +174,7 @@
                     'maintainer': doc.value['metadata']["DCTERMS.Creator"],
                     'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
                     'notes': html2text.html2text(doc.value['metadata']['Description']),
-                    'owner_org': org["result"]["id"]
+                    'owner_org': org_id
                     #todo add missing key values like jurasdiction
                 }
             if doc.value['agencyID'] == "qld":
@@ -201,10 +204,8 @@
             for group_name in groups:
                 group_url = name_munge(group_name[:100])
                 try:
-                    print ckan.group_entity_get(group_url)
-
                     # Update the group details
-                    group_entity = ckan.last_message
+                    group_entity = ckan.group_entity_get(group_url)
                     print "group "+group_name+" exists"
                     if 'packages' in group_entity.keys():
                         group_entity['packages'] = list(set(group_entity['packages'] + [pkg_name]))