beginning of export merge
[disclosr.git] / documents / datagov-export.py
blob:a/documents/datagov-export.py -> blob:b/documents/datagov-export.py
--- a/documents/datagov-export.py
+++ b/documents/datagov-export.py
@@ -12,8 +12,8 @@
 #ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api',    api_key='b47b24cd-591d-40c1-8677-d73101d56d1b')
 ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api',
     api_key='482a9dd2-a976-4adf-ac77-d71d92a98a52')
-#couch = couchdb.Server('http://127.0.0.1:5984/')
-couch = couchdb.Server('http://192.168.1.113:5984/')
+couch = couchdb.Server('http://127.0.0.1:5984/')
+#couch = couchdb.Server('http://192.168.1.113:5984/')
 
 # http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/
 SYMBOLS = {
@@ -91,6 +91,7 @@
 
 def name_munge(input_name):
     return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
+    #[:100]
     #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and')
 
 
@@ -117,32 +118,35 @@
 if __name__ == "__main__":
     for doc in docsdb.view('app/datasets'):
         print doc.id
-        if doc.value['url'] != "http://data.gov.au/data/":
+        if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
             # Collect the package metadata.
-            pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/','');                                                                  _
+            pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/',''); 
             tags = []
-            if len(doc.value['metadata']["Keywords / Tags"]) > 0:
-                if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'):
-                    tags = tags + doc.value['metadata']["Keywords / Tags"]
-                else:
-                    tags = tags + [doc.value['metadata']["Keywords / Tags"]]
-            if 'data.gov.au Category' in doc.value['metadata'].keys() and len(doc.value['metadata']['data.gov.au Category']) > 0:
-                if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'):
-                    tags = tags + doc.value['metadata']['data.gov.au Category']
-                else:
-                    tags = tags + [doc.value['metadata']['data.gov.au Category']]
-            tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag]
-            print tags
-            package_entity = {
-                'name': pkg_name,
-                'title': doc.value['metadata']['DCTERMS.Title'],
-                'url': doc.value['metadata']['DCTERMS.Source.URI'],
-                'tags': tags, #tags are mandatory?
-                'author': doc.value['metadata']["DCTERMS.Creator"],
-                'maintainer': doc.value['metadata']["DCTERMS.Creator"],
-                'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
-                'notes': html2text.html2text(doc.value['metadata']['Description']),
-            }
+            if doc.value['agencyID'] == "AGIMO":
+                if len(doc.value['metadata']["Keywords / Tags"]) > 0:
+                    if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'):
+                        tags = tags + doc.value['metadata']["Keywords / Tags"]
+                    else:
+                        tags = tags + [doc.value['metadata']["Keywords / Tags"]]
+                if 'data.gov.au Category' in doc.value['metadata'].keys() and len(doc.value['metadata']['data.gov.au Category']) > 0:
+                    if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'):
+                        tags = tags + doc.value['metadata']['data.gov.au Category']
+                    else:
+                        tags = tags + [doc.value['metadata']['data.gov.au Category']]
+                tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag]
+                print tags
+                package_entity = {
+                    'name': pkg_name,
+                    'title': doc.value['metadata']['DCTERMS.Title'],
+                    'url': doc.value['metadata']['DCTERMS.Source.URI'],
+                    'tags': tags, #tags are mandatory?
+                    'author': doc.value['metadata']["DCTERMS.Creator"],
+                    'maintainer': doc.value['metadata']["DCTERMS.Creator"],
+                    'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
+                    'notes': html2text.html2text(doc.value['metadata']['Description']),
+                }
+            if doc.value['agencyID'] == "qld":
+                package_entity = doc.value['metadata']
 
             try:
                 print package_entity
@@ -182,6 +186,8 @@
                         }
                     print group_entity
                     ckan.group_register_post(group_entity)
+                elif ckan.last_status == 409:
+		    print "group already exists"
                 else:
                     raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % (
                         ckan.last_status, pkg_name, e.args))