From: Alex Sadleir <maxious@lambdacomplex.org>
Date: Thu, 11 Apr 2013 12:22:35 +0000
Subject: datagov fixes
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=c64b3fe20debb60737f29c859d2bcf41ef0f70a7
---
datagov fixes


Former-commit-id: ed3ba96db4beeb126f802a3168476e27f298aeb8
---


--- a/documents/datagov-export.py
+++ b/documents/datagov-export.py
@@ -12,8 +12,8 @@
 #ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api',    api_key='b47b24cd-591d-40c1-8677-d73101d56d1b')
 ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api',
     api_key='482a9dd2-a976-4adf-ac77-d71d92a98a52')
-#couch = couchdb.Server('http://127.0.0.1:5984/')
-couch = couchdb.Server('http://192.168.1.113:5984/')
+couch = couchdb.Server('http://127.0.0.1:5984/')
+#couch = couchdb.Server('http://192.168.1.113:5984/')
 
 # http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/
 SYMBOLS = {
@@ -91,6 +91,7 @@
 
 def name_munge(input_name):
     return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
+    #[:100]
     #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and')
 
 
@@ -117,9 +118,9 @@
 if __name__ == "__main__":
     for doc in docsdb.view('app/datasets'):
         print doc.id
-        if doc.value['url'] != "http://data.gov.au/data/":
+        if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
             # Collect the package metadata.
-            pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/','');                                                                  _
+            pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/',''); 
             tags = []
             if doc.value['agencyID'] == "AGIMO":
                 if len(doc.value['metadata']["Keywords / Tags"]) > 0:
@@ -185,6 +186,8 @@
                         }
                     print group_entity
                     ckan.group_register_post(group_entity)
+                elif ckan.last_status == 409:
+		    print "group already exists"
                 else:
                     raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % (
                         ckan.last_status, pkg_name, e.args))

--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -201,7 +201,7 @@
     def getDate(self, content, entry, doc):
         date = ''.join(content.stripped_strings).strip()
         (a, b, c) = date.partition("(")
-        date = self.remove_control_chars(a.replace("Octber", "October").replace("1012","2012"))
+        date = self.remove_control_chars(a.replace("Octber", "October").replace("1012","2012").replace("Janurary","January"))
         print date
         edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
         print edate

--- a/documents/runScrapers.sh
+++ b/documents/runScrapers.sh
@@ -1,10 +1,14 @@
-for f in scrapers/*.py; 
-	do echo "Processing $f file.."; 
-	python $f; 
+rm /tmp/disclosr-error
+for f in scrapers/*.py; do
+	echo "Processing $f file..";
+	python $f 2>/tmp/disclosr-error;
 	if [ "$?" -ne "0" ]; then
 		echo "error";
-		sleep 2; 
+		sleep 2;
 	fi
 done
+if [ -s /tmp/disclosr-error ] ; then
+    mail -E -s "Disclosr errors" maxious@lambdacomplex.org < /tmp/disclosr-error ;
+fi
 
 

--- a/documents/scrape.py
+++ b/documents/scrape.py
@@ -13,8 +13,8 @@
 import socket
 
 #couch = couchdb.Server('http://192.168.1.148:5984/')
-couch = couchdb.Server('http://192.168.1.113:5984/')
-#couch = couchdb.Server('http://127.0.0.1:5984/')
+#couch = couchdb.Server('http://192.168.1.113:5984/')
+couch = couchdb.Server('http://127.0.0.1:5984/')
 
 
 def mkhash(input):