From: root <root@ip-172-31-18-206.ap-southeast-2.compute.internal>
Date: Wed, 28 May 2014 01:25:52 +0000
Subject: Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-datagovau.git&a=commitdiff&h=e70fa7eec6ad3c261af8ec034d2ea0d0c575cfa6
---
Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau
---


--- a/admin/autoupdate.py
+++ b/admin/autoupdate.py
@@ -1,18 +1,17 @@
 import requests
 import ckanapi
-# copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV;
-# f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}"
-import fileinput
 import csv
+import sys
+import psycopg2
+import json
+from subprocess import Popen, PIPE
 
-def updateresource(resource_id):
-    url = 'http://data.disclosurelo.gs'
-    api_key = ''
-    db_credentials = ''
-    print resource_id
-    ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs')
+def updateresource(resource_id, dataset_id):
+    print ' --- '
+    ckan = ckanapi.RemoteCKAN(api_url,api_key)
     #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org')
     resource = ckan.action.resource_show(id=resource_id)
+    print 'updating '+resource['name']+'('+resource_id+', '+dataset_id+')'
     print resource
     url = resource['url'] 
     #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT'
@@ -22,6 +21,7 @@
         headers['If-None-Match'] = resource['etag']
     if 'file_last_modified' in resource:
         headers["If-Modified-Since"] = resource['file_last_modified']
+    print headers
     r = requests.head(url, headers=headers)
     if r.status_code == 304:
         print 'not modified'
@@ -34,20 +34,42 @@
         if 'etag' in r.headers:
             resource['etag'] = r.headers['etag']
         #save updated resource
-#    result = ckan.action.resource_update(id,resource)
-        if 'format' == 'shp':
-            print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id']
+        print resource
+        result = ckan.call_action('resource_update',resource)
+        if resource['format'].lower() == 'shp' or resource['format'].lower() == 'kml':
+            print "geoingest!" 
+            pargs= ['dga-spatialingestor.py', db_settings_json, api_url, api_key, dataset_id]
+            print pargs
+            p = Popen(pargs)#, stdout=PIPE, stderr=PIPE)
+            p.communicate()
         else:
-            ckan.action.datapusher_submit(resource_id=id)
+            print "datapusher!"
+            # https://github.com/ckan/ckan/blob/master/ckanext/datapusher/logic/action.py#L19
+            ckan.action.datapusher_submit(resource_id=resource_id)
         
-if len(sys.argv) != 5:
-    print "spatial ingester. command line: postgis_url api_url api_key"
-    sys.exit(errno.EACCES)
+if len(sys.argv) != 4:
+    print "autoupdate ingester. command line: postgis_url api_url api_key"
+    sys.exit(-1)
 else:
     (path, db_settings_json, api_url, api_key) = sys.argv
     db_settings = json.loads(db_settings_json)
+    datastore_db_settings = dict(db_settings)
+    datastore_db_settings['dbname'] = db_settings['datastore_dbname']
+    datastore_db_settings_json = json.dumps(datastore_db_settings)
 
-for line in fileinput.input():
-    row = csv.reader(line)
-    updateresource(line.replace("\n",""))
+try:
+    conn = psycopg2.connect(dbname=db_settings['dbname'], user=db_settings['user'], password=db_settings['password'], host=db_settings['host'])
+except:
+    failure("I am unable to connect to the database.")
+# Open a cursor to perform database operations
+cur = conn.cursor()
+conn.set_isolation_level(0)
+cur.execute('select resource.id resource_id, package.id dataset_id  from resource inner join resource_group on resource.resource_group_id = resource_group.id inner join package on resource_group.package_id = package.id where resource.extras like \'%"autoupdate": "active"%\';')
+row = cur.fetchone() 
+while row is not None:
+    updateresource(row[0],row[1])
+    # process
+    row = cur.fetchone()
+cur.close()
+conn.close()