Merge branch 'master' of git+ssh://maxious.lambdacomplex.org/git/ckanext-datagovau
[ckanext-datagovau.git] / admin / autoupdate.py
Alex Sadleir


Alex Sadleir



Alex Sadleir
Alex Sadleir


Alex Sadleir
Alex Sadleir
Alex Sadleir
Alex Sadleir








Alex Sadleir
Alex Sadleir











Alex Sadleir







Alex Sadleir
Alex Sadleir


Alex Sadleir
Alex Sadleir


Alex Sadleir
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
import requests
import ckanapi
# copy (select id,url,format,extras from resource where extras like '%"autoupdate": "active"%') TO STDOUT WITH CSV;
# f759e4b6-723c-4863-8a26-1529d689cad8,http://data.gov.au/geoserver/geelong-roofprints-kml/wms?request=GetCapabilities,wms,"{""autoupdate"": ""active""}"
import fileinput
import csv
 
def updateresource(resource_id):
    url = 'http://data.disclosurelo.gs'
    api_key = ''
    db_credentials = ''
    print resource_id
    ckan = ckanapi.RemoteCKAN('http://data.disclosurelo.gs')
    #ckan = ckanapi.RemoteCKAN('http://demo.ckan.org')
    resource = ckan.action.resource_show(id=resource_id)
    print resource
    url = resource['url'] 
    #last_modified= 'Mon, 24 Feb 2014 01:48:29 GMT'
    #etag='"1393206509.38-638"'
    headers={}
    if 'etag' in resource:
        headers['If-None-Match'] = resource['etag']
    if 'file_last_modified' in resource:
        headers["If-Modified-Since"] = resource['file_last_modified']
    r = requests.head(url, headers=headers)
    if r.status_code == 304:
        print 'not modified'
        return
    else:
        print r.status_code
        print r.headers
        if 'last-modified' in r.headers:
            resource['file_last_modified'] = r.headers['last-modified']
        if 'etag' in r.headers:
            resource['etag'] = r.headers['etag']
        #save updated resource
#    result = ckan.action.resource_update(id,resource)
        if 'format' == 'shp':
            print "geoingest!" db_settings_json, api_url, api_key, resource['dataset_id']
        else:
            ckan.action.datapusher_submit(resource_id=id)
        
if len(sys.argv) != 5:
    print "spatial ingester. command line: postgis_url api_url api_key"
    sys.exit(errno.EACCES)
else:
    (path, db_settings_json, api_url, api_key) = sys.argv
    db_settings = json.loads(db_settings_json)
 
for line in fileinput.input():
    row = csv.reader(line)
    updateresource(line.replace("\n",""))