From: CKAN data.gov.au Date: Mon, 02 Dec 2013 01:32:12 +0000 Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/dga-spatialingestor X-Git-Url: https://maxious.lambdacomplex.org/git/?p=dga-spatialingestor.git&a=commitdiff&h=00b319780f3053f439b57dcd1afc203e9881e230 --- Merge branch 'master' of ssh://apples.lambdacomplex.org/git/dga-spatialingestor --- --- a/README.md +++ b/README.md @@ -27,3 +27,7 @@ INSERT into spatial_ref_sys (srid, auth_name, auth_srid, proj4text, srtext) values ( 96643, 'sr-org', 6643, '', 'PROJCS["Albers134",GEOGCS["GCS_GDA_1994",DATUM["D_GDA_1994",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]],PROJECTION["Albers"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",134.0],PARAMETER["Standard_Parallel_1",-18.0],PARAMETER["Standard_Parallel_2",-36.0],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]'); + +example run +python dga-spatialingestor.py '{"dbname":"geodatastore", "user":"postgres", "password":"snmc", "host":"localhost"}' http://localhost:5000 256fa905-cf92-4d6c-8714-95e3da2ea3c2 geodataset + --- a/dga-spatialingestor.py +++ b/dga-spatialingestor.py @@ -1,17 +1,26 @@ +#!/usr/bin/python +# coding=utf-8 +''' +spatial ingestor for data.gov.au + +1.0 28/11/2013 initial implementation +''' import ckanapi #https://github.com/open-data/ckanapi import errno, os, shutil, sys, glob from pprint import pprint from email.mime.text import MIMEText from subprocess import Popen, PIPE import tempfile +import smtplib from zipfile import ZipFile +from datetime import datetime import urllib import fileinput import json import psycopg2 import requests -geoserver_addr = "http://localhost:8080/geoserver/" +geoserver_addr = "http://data.disclosurelo.gs:8080/geoserver/" geoserver_user = "admin" geoserver_passwd = "geoserver" email_addr = "maxious@lambdacomplex.org" @@ -31,12 +40,12 @@ def success(msg): print "Completed!" -# email("geodata success",msg) + email("geodata success",msg) sys.exit(errno.EACCES) def failure(msg): print "ERROR -"+msg -# email("geodata error",str(sys.argv)+msg) + email("geodata error",str(sys.argv)+msg) sys.exit(errno.EACCES) def get_cursor(db_settings): @@ -106,7 +115,7 @@ print "Already up to date" sys.exit(0); -#email("geodata processing started for "+dataset['id'], str(sys.argv)) +email("geodata processing started for "+dataset['id'], str(sys.argv)) msg = "" #download resource to tmpfile @@ -181,11 +190,12 @@ # name layer after resource title ftdata = {'featureType':{'name':table_name, 'title': resource['name']}} +(minx,miny, maxx, maxy) = bbox.replace("BOX","").replace("(","").replace(")","").replace(","," ").split(" ") +bbox_obj = { 'minx': minx,'maxx': maxx,'miny': miny,'maxy': maxy } + if nativeCRS != None: ftdata['featureType']['nativeCRS'] = nativeCRS else: - (minx,miny, maxx, maxy) = bbox.replace("BOX","").replace("(","").replace(")","").replace(","," ").split(" ") - bbox_obj = { 'minx': minx,'maxx': maxx,'miny': miny,'maxy': maxy } ftdata['featureType']['nativeBoundingBox'] = bbox_obj ftdata['featureType']['latLonBoundingBox'] = bbox_obj ftdata['featureType']['srs'] = "EPSG:4326" @@ -196,11 +206,28 @@ #generate wms/wfs api links, kml, png resources and add to package print bgjson dataset['spatial'] = bgjson -# ckan.action.resource_update(id=resource['id'],url=resource['url'],name=resource['name'], last_modified=datetime.now().isoformat()) -#/geodatasetws/wms -#/geodatasetws/wfs -#http://cloudnine.lambdacomplex.org:8080/geoserver/tiger/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=tiger:tiger_roads&outputFormat=application/json -#http://cloudnine.lambdacomplex.org:8080/geoserver/tiger/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=tiger:tiger_roads&outputFormat=csv + +ws_addr = geoserver_addr+dataset['name']+"/" +for format in ['image/png','kml']: + url = ws_addr+"wms?request=GetMap&layers="+table_name+"&bbox="+bbox_obj['minx']+","+bbox_obj['miny']+","+bbox_obj['maxx']+","+bbox_obj['maxy']+"&width=512&height=512&format="+urllib.quote(format) + if format == "image/png": + dataset['resources'].append({"name":dataset['title'] + " Preview Image","description":"View overview image of this dataset" ,"format":format,"url":url, "last_modified": datetime.now().isoformat()}) + if format == "kml": + dataset['resources'].append({"name":dataset['title'] + " KML","description":"For use in web and desktop spatial data tools including Google Earth" ,"format":format,"url":url, "last_modified": datetime.now().isoformat()}) +for format in ['csv','json']: + url = ws_addr+"wfs?request=GetFeature&typeName="+table_name+"&outputFormat="+urllib.quote(format) + if format == "csv": + dataset['resources'].append({"name": dataset['title'] + " CSV","description":"For summary of the objects/data in this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()}) + if format == "json": + dataset['resources'].append({"name":dataset['title'] + " GeoJSON","description":"For use in web-based data visualisation of this collection","format":format,"url":url, "last_modified": datetime.now().isoformat()}) +dataset['resources'].append({"name":dataset['name'] + " - Preview this Dataset (WMS)","description":"View the data in this datasets online via web-based WMS viewer","format":"wms", + "url":ws_addr+"wms?request=GetCapabilities", "last_modified": datetime.now().isoformat()}) +dataset['resources'].append({"name":dataset['title'] + " WFS Link","description":"WFS Link for use of live data in Desktop GIS tools","format":"wfs", + "url":ws_addr+"wfs?request=GetCapabilities", "last_modified": datetime.now().isoformat()}) + +pprint(dataset) +ckan.action.package_update(id=dataset['id'],spatial=dataset['spatial'],resources=dataset['resources']) + #delete tempdir shutil.rmtree(tempdir)