From: Maxious Date: Fri, 10 Feb 2012 01:46:13 +0000 Subject: Change has fields to be arrays of URLs that confirm those assertions X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=3cd1b5bba7a0023bbb4cd1de61c2dbdda40af431 --- Change has fields to be arrays of URLs that confirm those assertions Former-commit-id: 83e592c1490220f07899eff9a51c768ee6067720 --- --- a/getAgency.php +++ b/getAgency.php @@ -58,9 +58,7 @@ echo ""; } echo" "; - } else if (strpos($key, "has") === 0) { - echo ""; - } else { + } else { echo ""; if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") { echo "view"; @@ -80,11 +78,9 @@ foreach ($defaultFields as $defaultField) { if (!isset($row[$defaultField])) { if ($schemas['agency']['properties'][$defaultField]['type'] == "string") { - if (strpos($defaultField, "has") === 0) { - $row[$defaultField] = "false"; - } else { + $row[$defaultField] = ""; - } + } if ($schemas['agency']['properties'][$defaultField]['type'] == "array") { @@ -124,7 +120,7 @@ } } - $mode = "view"; + $mode = "edit"; if ($mode == "edit") { $row = addDefaultFields(object_to_array($row)); } else { --- a/include/couchdb.inc.php +++ b/include/couchdb.inc.php @@ -26,14 +26,16 @@ emit(doc._id, doc.parentOrg); } };"; - $obj->views->byName->map = "function(doc) { + $obj->views->byName->map = 'function(doc) { + if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { emit(doc.name, doc._id); for (name in doc.otherNames) { -if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { +if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) { emit(doc.otherNames[name], doc._id); } } -};"; + } +};'; $obj->views->foiEmails->map = "function(doc) { emit(doc._id, doc.foiEmail); @@ -80,24 +82,39 @@ emit("total", 1); } }'; - $obj->views->score->map = 'if(!String.prototype.startsWith){ + $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){ String.prototype.startsWith = function (str) { return !this.indexOf(str); } } - +if(!String.prototype.endsWith){ + String.prototype.endsWith = function(suffix) { +     return this.indexOf(suffix, this.length - suffix.length) !== -1; + }; +} function(doc) { -count = 0; if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { for(var propName in doc) { - if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) { - count++ + if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") || propName.endsWith("URL"))) { + emit(propName, 1); } } - emit(count+doc._id, {id:doc._id, name: doc.name, score:count}); + emit("total", 1); } }'; - + $obj->views->scoreHas->reduce = 'function (key, values, rereduce) { + return sum(values); +}'; + $obj->views->fieldNames->map = ' +function(doc) { +for(var propName in doc) { + emit(propName, doc._id); + } + +}'; + $obj->views->fieldNames->reduce = 'function (key, values, rereduce) { + return values.length; +}'; // allow safe updates (even if slightly slower due to extra: rev-detection check). return $db->save($obj, true); } --- a/schemas/agency.json.php +++ b/schemas/agency.json.php @@ -24,17 +24,30 @@ "consultanciesURL" => Array("type" => "string", "required" => true, "x-title" => "Consultants Hired", "description" => ""), "legalExpenditureURL" => Array("type" => "string", "required" => true, "x-title" => "Legal Services Expenditure", "description" => "Legal Services Expenditure mandated by Legal Services Directions 2005"), "recordsListURL" => Array("type" => "string", "required" => true, "x-title" => "Files/Records Held", "description" => "Indexed lists of departmental and agency files, mandated by the Senate"), - "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => ""), - "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), + "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => "FOI Disclosure Log URL"), + "FOIDocumentsRSSURL" => Array("type" => "string", "required" => false, "x-title" => "RSS Feed of FOI Documents Released", "description" => "FOI Disclosure Log in RSS format"), + "hasFOIPDF" => Array("type" => "array", "required" => false, "x-title" => "Has FOI Documents Released in PDF", "description" => "FOI Disclosure Log contains any PDFs", + "items" => Array("type" => "string")), + "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), "appointmentsURL" => Array("type" => "string", "required" => true, "x-title" => "Agency Appointments/Boards", "description" => "Departmental and agency appointments and vacancies , mandated by the Senate"), "advertisingURL" => Array("type" => "string", "required" => true, "x-title" => "Approved Advertising Campaigns", "description" => " Agency advertising and public information projects, mandated by the Senate "), - "hasRSS" => Array("type" => "string", "required" => true, "x-title" => "Has RSS", "description" => ""), - "hasMailingList" => Array("type" => "string", "required" => true, "x-title" => "Has Mailing List", "description" => ""), - "hasTwitter" => Array("type" => "string", "required" => true, "x-title" => "Has Twitter", "description" => ""), - "hasFacebook" => Array("type" => "string", "required" => true, "x-title" => "Has Facebook", "description" => ""), - "hasYouTube" => Array("type" => "string", "required" => true, "x-title" => "Has YouTube", "description" => ""), - "hasFlickr" => Array("type" => "string", "required" => true, "x-title" => "Has Flickr", "description" => ""), - "hasCCBY" => Array("type" => "string", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution"), + "hasRSS" => Array("type" => "array", "required" => true, "x-title" => "Has RSS", "description" => ""), + "hasMailingList" => Array("type" => "array", "required" => true, "x-title" => "Has Mailing List", "description" => "", + "items" => Array("type" => "string")), + "hasTwitter" => Array("type" => "array", "required" => true, "x-title" => "Has Twitter", "description" => "", + "items" => Array("type" => "string")), + "hasFacebook" => Array("type" => "array", "required" => true, "x-title" => "Has Facebook", "description" => "", + "items" => Array("type" => "string")), + "hasYouTube" => Array("type" => "array", "required" => true, "x-title" => "Has YouTube", "description" => "", + "items" => Array("type" => "string")), + "hasFlickr" => Array("type" => "array", "required" => true, "x-title" => "Has Flickr", "description" => "", + "items" => Array("type" => "string")), + "hasCCBY" => Array("type" => "array", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution", + "items" => Array("type" => "string")), + "hasRestrictiveLicence" => Array("type" => "array","required" => true, "x-title" => "Has Restrictive Licence", "description" => "Has any page licenced under terms more restrictive than Crown Copyright", + "items" => Array("type" => "string")), + "hasCrownCopyright" => Array("type" => "array", "required" => true, "x-title" => "Has Standard Crown Copyright licence", "description" => "Has any page still licenced under the former Commonwealth Copyright Administration", + "items" => Array("type" => "string")), ), /* "org":{"type":"object", "properties":{ --- a/scrape.py +++ b/scrape.py @@ -3,6 +3,10 @@ import urllib2 from BeautifulSoup import BeautifulSoup import re +import hashlib +from urlparse import urljoin +import time +import os #http://diveintopython.org/http_web_services/etags.html class NotModifiedHandler(urllib2.BaseHandler): @@ -11,66 +15,102 @@ addinfourl.code = code return addinfourl -def scrapeAndStore(URL, depth, agency): - URL = "http://www.google.com" - req = urllib2.Request(URL) - etag = 'y' - last_modified = 'y' - #if there is a previous version sotred in couchdb, load caching helper tags - if etag: - req.add_header("If-None-Match", etag) - if last_modified: - req.add_header("If-Modified-Since", last_modified) +def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True): + hash = hashlib.md5(url).hexdigest() + req = urllib2.Request(url) + print "Fetching %s" % url + doc = docsdb.get(hash) + if doc == None: + doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName} + else: + if (time.time() - doc['page_scraped']) < 3600: + print "Uh oh, trying to scrape URL again too soon!" + last_attachment_fname = doc["_attachments"].keys()[-1] + last_attachment = docsdb.get_attachment(doc,last_attachment_fname) + return (doc['mime_type'],last_attachment) + if scrape_again == False: + print "Not scraping this URL again as requested" + return (None,None) + + time.sleep(3) # wait 3 seconds to give webserver time to recover + + #if there is a previous version stored in couchdb, load caching helper tags + if doc.has_key('etag'): + req.add_header("If-None-Match", doc['etag']) + if doc.has_key('last_modified'): + req.add_header("If-Modified-Since", doc['last_modified']) opener = urllib2.build_opener(NotModifiedHandler()) url_handle = opener.open(req) headers = url_handle.info() # the addinfourls have the .info() too - etag = headers.getheader("ETag") - last_modified = headers.getheader("Last-Modified") - web_server = headers.getheader("Server") - file_size = headers.getheader("Content-Length") - mime_type = headers.getheader("Content-Type") - + doc['etag'] = headers.getheader("ETag") + doc['last_modified'] = headers.getheader("Last-Modified") + doc['date'] = headers.getheader("Date") + doc['page_scraped'] = time.time() + doc['web_server'] = headers.getheader("Server") + doc['powered_by'] = headers.getheader("X-Powered-By") + doc['file_size'] = headers.getheader("Content-Length") + doc['mime_type'] = headers.getheader("Content-Type").split(";")[0] if hasattr(url_handle, 'code'): if url_handle.code == 304: print "the web page has not been modified" + return (None,None) else: - #do scraping - html = url_handle.read() - # http://www.crummy.com/software/BeautifulSoup/documentation.html - soup = BeautifulSoup(html) - links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) - for link in links: - print link['href'] - #for each unique link - #if html mimetype - # go down X levels, - # diff with last stored attachment, store in document - #if not - # remember to save parentURL and title (link text that lead to document) - + content = url_handle.read() + docsdb.save(doc) + doc = docsdb.get(hash) # need to get a _rev + docsdb.put_attachment(doc, content, str(time.time())+"-"+os.path.basename(url), doc['mime_type']) + return (doc['mime_type'], content) #store as attachment epoch-filename else: - print "error %s in downloading %s", url_handle.code, URL - #record/alert error to error database - - + print "error %s in downloading %s" % url_handle.code, URL + doc['error'] = "error %s in downloading %s" % url_handle.code, URL + docsdb.save(doc) + return (None,None) +def scrapeAndStore(docsdb, url, depth, fieldName, agencyID): + (mime_type,content) = fetchURL(docsdb, url, fieldName, agencyID) + if content != None and depth > 0: + if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": + # http://www.crummy.com/software/BeautifulSoup/documentation.html + soup = BeautifulSoup(content) + navIDs = soup.findAll(id=re.compile('nav|Nav|menu|bar')) + for nav in navIDs: + print "Removing element", nav['id'] + nav.extract() + navClasses = soup.findAll(attrs={'class' : re.compile('nav|menu|bar')}) + for nav in navClasses: + print "Removing element", nav['class'] + nav.extract() + links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) + linkurls = set([]) + for link in links: + if link.has_key("href"): + if link['href'].startswith("http"): + # lets not do external links for now + # linkurls.add(link['href']) + None + else: + linkurls.add(urljoin(url,link['href'].replace(" ","%20"))) + for linkurl in linkurls: + #print linkurl + scrapeAndStore(docsdb, linkurl, depth-1, fieldName, agencyID) - - - - - -couch = couchdb.Server('http://192.168.1.148:5984/') +couch = couchdb.Server('http://127.0.0.1:5984/') # select database agencydb = couch['disclosr-agencies'] +docsdb = couch['disclosr-documents'] for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view? agency = agencydb.get(row.id) print agency['name'] -scrapeAndStore("A",1,1) + for key in agency.keys(): + if key == 'website' or key.endswith('URL'): + print key + scrapeAndStore(docsdb, agency[key],agency['scrapeDepth'],key,agency['_id']) + agency['metadata']['lastscraped'] = time.time() + agencydb.save(agency)