| import requests | import requests |
| import ckanapi | import ckanapi |
| # copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; | # copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV; |
| # f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" | # f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}" |
| import fileinput | import fileinput |
| import csv | import csv |
| def updateresource(id): | def updateresource(resource_id): |
| url = 'http://data.disclosurelo.gs' | url = 'http://data.disclosurelo.gs' |
| api_key = '' | api_key = '' |
| db_credentials = '' | db_credentials = '' |
| print id | print resource_id |
| ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') | ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs') |
| #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') | #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org') |
| resource = ckan.action.resource_show(id=id) | resource = ckan.action.resource_show(id=resource_id) |
| print resource | print resource |
| url = resource['url'] | url = resource['url'] |
| #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' | #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT' |
| #etag='"1393206509.38-638"' | #etag='"1393206509.38-638"' |
| headers={} | headers={} |
| if 'etag' in resource: | if 'etag' in resource: |
| headers['If-None-Match'] = resource['etag'] | headers['If-None-Match'] = resource['etag'] |
| if 'file_last_modified' in resource: | if 'file_last_modified' in resource: |
| headers["If-Modified-Since"] = resource['file_last_modified'] | headers["If-Modified-Since"] = resource['file_last_modified'] |
| r = requests.head(url, headers=headers) | r = requests.head(url, headers=headers) |
| if r.status_code == 304: | if r.status_code == 304: |
| print 'not modified' | print 'not modified' |
| return | return |
| else: | else: |
| print r.status_code | print r.status_code |
| print r.headers | print r.headers |
| if 'last-modified' in r.headers: | if 'last-modified' in r.headers: |
| resource['file_last_modified'] = r.headers['last-modified'] | resource['file_last_modified'] = r.headers['last-modified'] |
| if 'etag' in r.headers: | if 'etag' in r.headers: |
| resource['etag'] = r.headers['etag'] | resource['etag'] = r.headers['etag'] |
| #save updated resource | #save updated resource |
| # result = ckan.action.resource_update(id,resource) | |
| if 'format' == 'shp': | if 'format' == 'shp': |
| print "geoingest!" | print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id'] |
| else: | else: |
| ckan.action.datapusher_submit(resource_id=id) | ckan.action.datapusher_submit(resource_id=id) |
| # result = ckan.action.resource_update(id,resource) | |
| if len(sys.argv) != 5: | |
| print "spatial ingester. command line: postgis_url api_url api_key" | |
| sys.exit(errno.EACCES) | |
| else: | |
| (path, db_settings_json, api_url, api_key) = sys.argv | |
| db_settings = json.loads(db_settings_json) | |
| for line in fileinput.input(): | for line in fileinput.input(): |
| row = csv.reader(line) | row = csv.reader(line) |
| updateresource(line.replace("\n","")) | updateresource(line.replace("\n","")) |