Merge branch 'master' of ssh://apples.lambdacomplex.org/git/dga-spatialingestor
--- a/README.md
+++ b/README.md
@@ -27,3 +27,7 @@
INSERT into spatial_ref_sys (srid, auth_name, auth_srid, proj4text, srtext) values ( 96643, 'sr-org', 6643, '', 'PROJCS["Albers134",GEOGCS["GCS_GDA_1994",DATUM["D_GDA_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]],PROJECTION["Albers"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",134.0],PARAMETER["Standard_Parallel_1",-18.0],PARAMETER["Standard_Parallel_2",-36.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]');
+
+example run
+python dga-spatialingestor.py '{"dbname":"geodatastore", "user":"postgres", "password":"snmc", "host":"localhost"}' http://localhost:5000 256fa905-cf92-4d6c-8714-95e3da2ea3c2 geodataset
+
--- a/dga-spatialingestor.py
+++ b/dga-spatialingestor.py
@@ -1,17 +1,26 @@
+#!/usr/bin/python
+# coding=utf-8
+'''
+spatial ingestor for data.gov.au
+<alex.sadleir@linkdigital.com.au>
+1.0 28/11/2013 initial implementation
+'''
import ckanapi #https://github.com/open-data/ckanapi
import errno, os, shutil, sys, glob
from pprint import pprint
from email.mime.text import MIMEText
from subprocess import Popen, PIPE
import tempfile
+import smtplib
from zipfile import ZipFile
+from datetime import datetime
import urllib
import fileinput
import json
import psycopg2
import requests
-geoserver_addr = "http://localhost:8080/geoserver/"
+geoserver_addr = "http://data.disclosurelo.gs:8080/geoserver/"
geoserver_user = "admin"
geoserver_passwd = "geoserver"
email_addr = "maxious@lambdacomplex.org"
@@ -31,12 +40,12 @@
def success(msg):
print "Completed!"
-# email("geodata success",msg)
+ email("geodata success",msg)
sys.exit(errno.EACCES)
def failure(msg):
print "ERROR -"+msg
-# email("geodata error",str(sys.argv)+msg)
+ email("geodata error",str(sys.argv)+msg)
sys.exit(errno.EACCES)
def get_cursor(db_settings):
@@ -106,7 +115,7 @@
print "Already up to date"
sys.exit(0);
-#email("geodata processing started for "+dataset['id'], str(sys.argv))
+email("geodata processing started for "+dataset['id'], str(sys.argv))
msg = ""
#download resource to tmpfile
@@ -181,11 +190,12 @@
# name layer after resource title
ftdata = {'featureType':{'name':table_name, 'title': resource['name']}}
+(minx,miny, maxx, maxy) = bbox.replace("BOX","").replace("(","").replace(")","").replace(","," ").split(" ")
+bbox_obj = { 'minx': minx,'maxx': maxx,'miny': miny,'maxy': maxy }
+
if nativeCRS != None:
ftdata['featureType']['nativeCRS'] = nativeCRS
else:
- (minx,miny, maxx, maxy) = bbox.replace("BOX","").replace("(","").replace(")","").replace(","," ").split(" ")
- bbox_obj = { 'minx': minx,'maxx': maxx,'miny': miny,'maxy': maxy }
ftdata['featureType']['nativeBoundingBox'] = bbox_obj
ftdata['featureType']['latLonBoundingBox'] = bbox_obj
ftdata['featureType']['srs'] = "EPSG:4326"
@@ -196,11 +206,28 @@
#generate wms/wfs api links, kml, png resources and add to package
print bgjson
dataset['spatial'] = bgjson
-# ckan.action.resource_update(id=resource['id'],url=resource['url'],name=resource['name'], last_modified=datetime.now().isoformat())
-#/geodatasetws/wms
-#/geodatasetws/wfs
-#http://cloudnine.lambdacomplex.org:8080/geoserver/tiger/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=tiger:tiger_roads&outputFormat=application/json
-#http://cloudnine.lambdacomplex.org:8080/geoserver/tiger/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=tiger:tiger_roads&outputFormat=csv
+
+ws_addr = geoserver_addr+dataset['name']+"/"
+for format in ['image/png','kml']:
+ url = ws_addr+"wms?request=GetMap&layers="+table_name+"&bbox="+bbox_obj['minx']+","+bbox_obj['miny']+","+bbox_obj['maxx']+","+bbox_obj['maxy']+"&width=512&height=512&format="+urllib.quote(format)
+ if format == "image/png":
+ dataset['resources'].append({"name":dataset['title'] + " Preview Image","description":"View overview image of this dataset" ,"format":format,"url":url, "last_modified": datetime.now().isoformat()})
+ if format == "kml":
+ dataset['resources'].append({"name":dataset['title'] + " KML","description":"For use in web and desktop spatial data tools including Google Earth" ,"format":format,"url":url, "last_modified": datetime.now().isoformat()})
+for format in ['csv','json']:
+ url = ws_addr+"wfs?request=GetFeature&typeName="+table_name+"&outputFormat="+urllib.quote(format)
+ if format == "csv":
+ dataset['resources'].append({"name": dataset['title'] + " CSV","description":"For summary of the objects/data in this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()})
+ if format == "json":
+ dataset['resources'].append({"name":dataset['title'] + " GeoJSON","description":"For use in web-based data visualisation of this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()})
+dataset['resources'].append({"name":dataset['name'] + " - Preview this Dataset (WMS)","description":"View the data in this datasets online via web-based WMS viewer","format":"wms",
+ "url":ws_addr+"wms?request=GetCapabilities", "last_modified": datetime.now().isoformat()})
+dataset['resources'].append({"name":dataset['title'] + " WFS Link","description":"WFS Link for use of live data in Desktop GIS tools","format":"wfs",
+ "url":ws_addr+"wfs?request=GetCapabilities", "last_modified": datetime.now().isoformat()})
+
+pprint(dataset)
+ckan.action.package_update(id=dataset['id'],spatial=dataset['spatial'],resources=dataset['resources'])
+
#delete tempdir
shutil.rmtree(tempdir)