From: CKAN data.gov.au <ckandga@ip-172-31-18-206.ap-southeast-2.compute.internal>
Date: Mon, 02 Dec 2013 06:06:13 +0000
Subject: Add KML/KMZ support
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=dga-spatialingestor.git&a=commitdiff&h=e8d573cf05f51c66075230e40353a2856fabc18f
---
Add KML/KMZ support
---


--- a/dga-spatialingestor.py
+++ b/dga-spatialingestor.py
@@ -19,6 +19,8 @@
 import json
 import psycopg2
 import requests
+from dateutil import parser
+import lxml.etree as et
 
 geoserver_addr = "http://data.gov.au/geoserver/"
 geoserver_user = "admin"
@@ -77,6 +79,7 @@
 		cur.execute(sql)
 		sql = ""
 	if sql != "":
+		#print sql
 		cur.execute(sql)
         cur.close()
         conn.close()
@@ -107,19 +110,19 @@
 		else:
 			ows_resources += [resource]
 		
-#	if "kml" in resource['format']:
-#		data_modified_date = resource['last_modified']
-#		kml_resources += [resource]
+	if "kml" in resource['format'] or "kmz" in resource['format']:
+		data_modified_date = resource['last_modified']
+		kml_resources += [resource]
 	if "shp" in resource['format']:
 		data_modified_date = resource['last_modified']
 		shp_resources += [resource]
 
-if len(shp_resources) == 0:
+if len(shp_resources) + len(kml_resources) == 0:
 	print "No geodata format files detected"
 	sys.exit(0);
 
 #if geoserver api link does not exist or api link is out of date with data, continue
-if len(ows_resources) > 0 and data_modified_date <= wms_resources[0]['last_modified']:
+if len(ows_resources) > 0 and parser.parse(data_modified_date) <= wms_resources[0]['last_modified']:
 	print "Already up to date"
         sys.exit(0);
 
@@ -130,8 +133,7 @@
 #check filesize limit
 
 (cur,conn) = get_cursor(db_settings)
-table_name = dataset['id']
-#.replace("-","_")
+table_name = dataset['id'].replace("-","_")
 cur.execute('DROP TABLE IF EXISTS "'+table_name+'"')
 cur.close()
 conn.close()
@@ -151,16 +153,39 @@
 	prjfiles = glob.glob("*.[pP][rR][jJ]")
 	if len(shpfiles) == 0:
 		failure("no shp files found in zip "+shp_resources[0]['url'])
-	print "converting to pgsql "+shpfiles[0]
+	print "converting to pgsql "+table_name+" "+shpfiles[0]
 	process = Popen([shp2pgsql,shpfiles[0], table_name], stdout=PIPE, stderr=PIPE)
 	psql_load(process)
 	if len(prjfiles) > 0:
 		nativeCRS = open(prjfiles[0], 'r').read()
-#else:
-#	print "using KML file "+kml_resources[0]['url']
-#	#if kml ogr2ogr http://gis.stackexchange.com/questions/33102/how-to-import-kml-file-with-custom-data-to-postgres-postgis-database
-#	(filepath,headers) = urllib.urlretrieve(kml_resources[0]['url'], "input.kml")
-
+else:
+	print "using KML file "+kml_resources[0]['url']
+	nativeCRS = None
+	#if kml ogr2ogr http://gis.stackexchange.com/questions/33102/how-to-import-kml-file-with-custom-data-to-postgres-postgis-database
+	if kml_resources[0]['format'] == "kmz":
+		(filepath,headers) = urllib.urlretrieve(kml_resources[0]['url'], "input.zip" )
+		with ZipFile(filepath, 'r') as myzip:
+			myzip.extractall()
+		print "kmz unziped"
+		kmlfiles = glob.glob("*.[kK][mM][lL]")
+		if len(kmlfiles) == 0:
+			failure("no kml files found in zip "+kml_resources[0]['url'])
+		else:
+			kml_file = kmlfiles[0]
+	else: 
+		(filepath,headers) = urllib.urlretrieve(kml_resources[0]['url'], "input.kml")
+		kml_file = "input.kml"
+	print "changing kml folder name"
+	tree = et.parse(kml_file)
+	element = tree.xpath('//kml:Folder/kml:name', namespaces={'kml': "http://www.opengis.net/kml/2.2"})
+	element[0].text = table_name
+	with open(table_name+".kml", "w") as ofile: 
+		ofile.write(et.tostring(tree))
+	print "converting to pgsql "+table_name+".kml"
+	pargs = ['ogr2ogr','-f','PostgreSQL',"--config" ,"PG_USE_COPY","YES",'PG:dbname=\''+ db_settings['dbname']+'\' host=\''+db_settings['host']+'\' user=\''+db_settings['user']+ '\' password=\''+db_settings['password']+'\'' ,table_name+".kml",'-lco','GEOMETRY_NAME=geom']
+	pprint(pargs)
+	p = Popen(pargs)#, stdout=PIPE, stderr=PIPE)
+	p.communicate()
 
 #load bounding boxes
 (cur,conn) = get_cursor(db_settings)
@@ -218,6 +243,7 @@
 print bgjson
 dataset['spatial'] = bgjson
 
+#TODO update or append
 ws_addr = geoserver_addr+dataset['name']+"/"
 for format in ['image/png','kml']:
       url = ws_addr+"wms?request=GetMap&layers="+table_name+"&bbox="+bbox_obj['minx']+","+bbox_obj['miny']+","+bbox_obj['maxx']+","+bbox_obj['maxy']+"&width=512&height=512&format="+urllib.quote(format)