projects / dga-spatialingestor.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
side by side | plain
data.gov.au customisation
data.gov.au customisation

file:a/README.md -> file:b/README.md 
--- a/README.md
+++ b/README.md
@@ -24,6 +24,8 @@
   sudo -u postgres psql $db -c "CREATE EXTENSION postgis;"
 
   sudo -u postgres psql $db -c "ALTER TABLE geometry_columns OWNER TO $owner; ALTER TABLE spatial_ref_sys OWNER TO $owner;"
+grant select on table geometry_columns to ckandga_data; # grant select to read only user
+
 
 INSERT into spatial_ref_sys (srid, auth_name, auth_srid, proj4text, srtext) values ( 96643, 'sr-org', 6643, '', 'PROJCS["Albers134",GEOGCS["GCS_GDA_1994",DATUM["D_GDA_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]],PROJECTION["Albers"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",134.0],PARAMETER["Standard_Parallel_1",-18.0],PARAMETER["Standard_Parallel_2",-36.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]');
file:a/dga-spatialingestor.py -> file:b/dga-spatialingestor.py 
--- a/dga-spatialingestor.py
+++ b/dga-spatialingestor.py
@@ -20,21 +20,27 @@
 import psycopg2
 import requests
 
-geoserver_addr = "http://data.disclosurelo.gs:8080/geoserver/"
+geoserver_addr = "http://data.gov.au/geoserver/"
 geoserver_user = "admin"
-geoserver_passwd = "geoserver"
-email_addr = "maxious@lambdacomplex.org"
-shp2pgsql = "/usr/lib/postgresql/9.2/bin/shp2pgsql"
-omitted_orgs = []
+geoserver_passwd = "oRu7chan"
+email_addr = "alex.sadleir@linkdigital.com.au"
+shp2pgsql = "/usr/bin/shp2pgsql"
+omitted_orgs = ['launcestoncitycouncil','gcc']
 
 def email(subject, body):
 	msg = MIMEText(body)
-	msg["From"] = "ckan@localhost"
+	msg["From"] = "datagovau@gmail.com"
 	msg["To"] = email_addr
 	msg["Subject"] = subject
 	# Send the message via our own SMTP server, but don't include the
 	# envelope header.
-	s = smtplib.SMTP('localhost')
+	#p = Popen(["/usr/sbin/sendmail", "-t"], stdin=PIPE)
+	#p.communicate(msg.as_string())
+	s = smtplib.SMTP('smtp.gmail.com',587)
+	s.ehlo()
+	s.starttls()
+	s.ehlo
+	s.login('datagovau@gmail.com','3P4ELm9kjNAmKUL')
 	s.sendmail(msg["From"], [msg["To"]], msg.as_string())
 	s.quit()
 
@@ -86,8 +92,8 @@
 dataset = ckan.action.package_show(id=dataset_id)
 print "loaded dataset"+dataset['name']
 #pprint(dataset)
-if dataset['owner_org'] in omitted_orgs:
-	print(dataset.owner_org + " in omitted_orgs")
+if dataset['organization']['name'] in omitted_orgs:
+	print(dataset['organization']['name'] + " in omitted_orgs")
 	sys.exit(0);
 
 ows_resources = []
@@ -101,9 +107,11 @@
 		else:
 			ows_resources += [resource]
 		
-	if "kml" in resource['format']:
-		kml_resources += [resource]
+#	if "kml" in resource['format']:
+#		data_modified_date = resource['last_modified']
+#		kml_resources += [resource]
 	if "shp" in resource['format']:
+		data_modified_date = resource['last_modified']
 		shp_resources += [resource]
 
 if len(shp_resources) == 0:
@@ -115,15 +123,16 @@
 	print "Already up to date"
         sys.exit(0);
 
-email("geodata processing started for "+dataset['id'], str(sys.argv))
-msg = ""
+email("geodata processing started for "+dataset['id'], "")
+msg = dataset['id']
 #download resource to tmpfile
 
 #check filesize limit
 
 (cur,conn) = get_cursor(db_settings)
-table_name = dataset['id'].replace("-","_")
-cur.execute("DROP TABLE IF EXISTS "+table_name)
+table_name = dataset['id']
+#.replace("-","_")
+cur.execute('DROP TABLE IF EXISTS "'+table_name+'"')
 cur.close()
 conn.close()
 
@@ -155,7 +164,7 @@
 
 #load bounding boxes
 (cur,conn) = get_cursor(db_settings)
-cur.execute('SELECT ST_Extent(geom) as box,ST_AsGeoJSON(ST_Extent(geom)) as geojson from '+table_name)
+cur.execute('SELECT ST_Extent(geom) as box,ST_AsGeoJSON(ST_Extent(geom)) as geojson from "'+table_name+'"')
 (bbox,bgjson) = cur.fetchone()
 cur.close()
 conn.close()
@@ -165,7 +174,7 @@
 #create geoserver dataset http://boundlessgeo.com/2012/10/adding-layers-to-geoserver-using-the-rest-api/
 # name workspace after dataset
 workspace = dataset['name']
-ws = requests.post(geoserver_addr+'rest/workspaces', data=json.dumps({'workspace': {'name': workspace} }), headers={'Content-type': 'application/json'}, auth=('admin', 'geoserver'))
+ws = requests.post(geoserver_addr+'rest/workspaces', data=json.dumps({'workspace': {'name': workspace} }), headers={'Content-type': 'application/json'}, auth=(geoserver_user, geoserver_passwd))
 pprint(ws)
 #echo ws.status_code
 #echo ws.text
@@ -177,19 +186,19 @@
      'port':5432,
      'database': db_settings['dbname'],
      'schema':'public',
-     'user':db_settings['user'],
+     'user':db_settings['user'] + "_data", #use read only user
      'passwd':db_settings['password'],
      'dbtype':'postgis'
 
                 }}})
-#print dsdata
-r = requests.post(geoserver_addr+'rest/workspaces/'+workspace+'/datastores', data=dsdata, headers={'Content-type': 'application/json'}, auth=('admin', 'geoserver'))
+print dsdata
+r = requests.post(geoserver_addr+'rest/workspaces/'+workspace+'/datastores', data=dsdata, headers={'Content-type': 'application/json'}, auth=(geoserver_user, geoserver_passwd))
 pprint(r)
 #echo r.status_code
 #echo r.text
 
 # name layer after resource title
-ftdata = {'featureType':{'name':table_name, 'title': resource['name']}}
+ftdata = {'featureType':{'name':table_name, 'title': dataset['title']}}
 (minx,miny, maxx, maxy) = bbox.replace("BOX","").replace("(","").replace(")","").replace(","," ").split(" ")
 bbox_obj = { 'minx': minx,'maxx': maxx,'miny': miny,'maxy': maxy }
 
@@ -200,7 +209,9 @@
 	ftdata['featureType']['latLonBoundingBox'] = bbox_obj
 	ftdata['featureType']['srs'] = "EPSG:4326"
 ftdata = json.dumps(ftdata)
-r = requests.post(geoserver_addr+'rest/workspaces/'+workspace+'/datastores/'+datastore+"/featuretypes", data= ftdata, headers={'Content-Type': 'application/json'}, auth=('admin', 'geoserver'))
+print geoserver_addr+'rest/workspaces/'+workspace+'/datastores/'+datastore+"/featuretypes"
+print ftdata
+r = requests.post(geoserver_addr+'rest/workspaces/'+workspace+'/datastores/'+datastore+"/featuretypes", data= ftdata, headers={'Content-Type': 'application/json'}, auth=(geoserver_user, geoserver_passwd))
 pprint(r)
 
 #generate wms/wfs api links, kml, png resources and add to package
@@ -220,13 +231,13 @@
                dataset['resources'].append({"name": dataset['title'] + " CSV","description":"For summary of the objects/data in this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()})
        if format == "json":
                dataset['resources'].append({"name":dataset['title'] + " GeoJSON","description":"For use in web-based data visualisation of this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()})
-dataset['resources'].append({"name":dataset['name'] + " - Preview this Dataset (WMS)","description":"View the data in this datasets online via web-based WMS viewer","format":"wms",
+dataset['resources'].append({"name":dataset['title'] + " - Preview this Dataset (WMS)","description":"View the data in this datasets online via web-based WMS viewer","format":"wms",
     "url":ws_addr+"wms?request=GetCapabilities", "last_modified": datetime.now().isoformat()})
 dataset['resources'].append({"name":dataset['title'] + " WFS Link","description":"WFS Link for use of live data in Desktop GIS tools","format":"wfs",
     "url":ws_addr+"wfs?request=GetCapabilities", "last_modified": datetime.now().isoformat()})
 
 pprint(dataset)
-ckan.action.package_update(id=dataset['id'],spatial=dataset['spatial'],resources=dataset['resources'])
+#ckan.action.package_update(id=dataset['id'],spatial=dataset['spatial'],resources=dataset['resources'])
 
 
 #delete tempdir
Unnamed repository; edit this file 'description' to name the repository.
GitPHP by Chris Han