From: Maxious Date: Fri, 10 Feb 2012 01:46:13 +0000 Subject: Change has fields to be arrays of URLs that confirm those assertions X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=3cd1b5bba7a0023bbb4cd1de61c2dbdda40af431 --- Change has fields to be arrays of URLs that confirm those assertions Former-commit-id: 83e592c1490220f07899eff9a51c768ee6067720 --- --- a/getAgency.php +++ b/getAgency.php @@ -58,9 +58,7 @@ echo ""; } echo" "; - } else if (strpos($key, "has") === 0) { - echo ""; - } else { + } else { echo ""; if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") { echo "view"; @@ -80,11 +78,9 @@ foreach ($defaultFields as $defaultField) { if (!isset($row[$defaultField])) { if ($schemas['agency']['properties'][$defaultField]['type'] == "string") { - if (strpos($defaultField, "has") === 0) { - $row[$defaultField] = "false"; - } else { + $row[$defaultField] = ""; - } + } if ($schemas['agency']['properties'][$defaultField]['type'] == "array") { @@ -124,7 +120,7 @@ } } - $mode = "view"; + $mode = "edit"; if ($mode == "edit") { $row = addDefaultFields(object_to_array($row)); } else { --- a/include/couchdb.inc.php +++ b/include/couchdb.inc.php @@ -26,14 +26,16 @@ emit(doc._id, doc.parentOrg); } };"; - $obj->views->byName->map = "function(doc) { + $obj->views->byName->map = 'function(doc) { + if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { emit(doc.name, doc._id); for (name in doc.otherNames) { -if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { +if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) { emit(doc.otherNames[name], doc._id); } } -};"; + } +};'; $obj->views->foiEmails->map = "function(doc) { emit(doc._id, doc.foiEmail); @@ -80,24 +82,39 @@ emit("total", 1); } }'; - $obj->views->score->map = 'if(!String.prototype.startsWith){ + $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){ String.prototype.startsWith = function (str) { return !this.indexOf(str); } } - +if(!String.prototype.endsWith){ + String.prototype.endsWith = function(suffix) { +     return this.indexOf(suffix, this.length - suffix.length) !== -1; + }; +} function(doc) { -count = 0; if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { for(var propName in doc) { - if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) { - count++ + if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") || propName.endsWith("URL"))) { + emit(propName, 1); } } - emit(count+doc._id, {id:doc._id, name: doc.name, score:count}); + emit("total", 1); } }'; - + $obj->views->scoreHas->reduce = 'function (key, values, rereduce) { + return sum(values); +}'; + $obj->views->fieldNames->map = ' +function(doc) { +for(var propName in doc) { + emit(propName, doc._id); + } + +}'; + $obj->views->fieldNames->reduce = 'function (key, values, rereduce) { + return values.length; +}'; // allow safe updates (even if slightly slower due to extra: rev-detection check). return $db->save($obj, true); } --- a/schemas/agency.json.php +++ b/schemas/agency.json.php @@ -24,17 +24,30 @@ "consultanciesURL" => Array("type" => "string", "required" => true, "x-title" => "Consultants Hired", "description" => ""), "legalExpenditureURL" => Array("type" => "string", "required" => true, "x-title" => "Legal Services Expenditure", "description" => "Legal Services Expenditure mandated by Legal Services Directions 2005"), "recordsListURL" => Array("type" => "string", "required" => true, "x-title" => "Files/Records Held", "description" => "Indexed lists of departmental and agency files, mandated by the Senate"), - "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => ""), - "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), + "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => "FOI Disclosure Log URL"), + "FOIDocumentsRSSURL" => Array("type" => "string", "required" => false, "x-title" => "RSS Feed of FOI Documents Released", "description" => "FOI Disclosure Log in RSS format"), + "hasFOIPDF" => Array("type" => "array", "required" => false, "x-title" => "Has FOI Documents Released in PDF", "description" => "FOI Disclosure Log contains any PDFs", + "items" => Array("type" => "string")), + "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), "appointmentsURL" => Array("type" => "string", "required" => true, "x-title" => "Agency Appointments/Boards", "description" => "Departmental and agency appointments and vacancies , mandated by the Senate"), "advertisingURL" => Array("type" => "string", "required" => true, "x-title" => "Approved Advertising Campaigns", "description" => " Agency advertising and public information projects, mandated by the Senate "), - "hasRSS" => Array("type" => "string", "required" => true, "x-title" => "Has RSS", "description" => ""), - "hasMailingList" => Array("type" => "string", "required" => true, "x-title" => "Has Mailing List", "description" => ""), - "hasTwitter" => Array("type" => "string", "required" => true, "x-title" => "Has Twitter", "description" => ""), - "hasFacebook" => Array("type" => "string", "required" => true, "x-title" => "Has Facebook", "description" => ""), - "hasYouTube" => Array("type" => "string", "required" => true, "x-title" => "Has YouTube", "description" => ""), - "hasFlickr" => Array("type" => "string", "required" => true, "x-title" => "Has Flickr", "description" => ""), - "hasCCBY" => Array("type" => "string", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution"), + "hasRSS" => Array("type" => "array", "required" => true, "x-title" => "Has RSS", "description" => ""), + "hasMailingList" => Array("type" => "array", "required" => true, "x-title" => "Has Mailing List", "description" => "", + "items" => Array("type" => "string")), + "hasTwitter" => Array("type" => "array", "required" => true, "x-title" => "Has Twitter", "description" => "", + "items" => Array("type" => "string")), + "hasFacebook" => Array("type" => "array", "required" => true, "x-title" => "Has Facebook", "description" => "", + "items" => Array("type" => "string")), + "hasYouTube" => Array("type" => "array", "required" => true, "x-title" => "Has YouTube", "description" => "", + "items" => Array("type" => "string")), + "hasFlickr" => Array("type" => "array", "required" => true, "x-title" => "Has Flickr", "description" => "", + "items" => Array("type" => "string")), + "hasCCBY" => Array("type" => "array", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution", + "items" => Array("type" => "string")), + "hasRestrictiveLicence" => Array("type" => "array","required" => true, "x-title" => "Has Restrictive Licence", "description" => "Has any page licenced under terms more restrictive than Crown Copyright", + "items" => Array("type" => "string")), + "hasCrownCopyright" => Array("type" => "array", "required" => true, "x-title" => "Has Standard Crown Copyright licence", "description" => "Has any page still licenced under the former Commonwealth Copyright Administration", + "items" => Array("type" => "string")), ), /* "org":{"type":"object", "properties":{ --- a/scrape.py +++ b/scrape.py @@ -4,6 +4,9 @@ from BeautifulSoup import BeautifulSoup import re import hashlib +from urlparse import urljoin +import time +import os #http://diveintopython.org/http_web_services/etags.html class NotModifiedHandler(urllib2.BaseHandler): @@ -12,11 +15,25 @@ addinfourl.code = code return addinfourl -def scrapeAndStore(docsdb, url, depth, agencyID): +def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True): hash = hashlib.md5(url).hexdigest() req = urllib2.Request(url) - print "Fetching %s", url - doc = docsdb['hash'] + print "Fetching %s" % url + doc = docsdb.get(hash) + if doc == None: + doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName} + else: + if (time.time() - doc['page_scraped']) < 3600: + print "Uh oh, trying to scrape URL again too soon!" + last_attachment_fname = doc["_attachments"].keys()[-1] + last_attachment = docsdb.get_attachment(doc,last_attachment_fname) + return (doc['mime_type'],last_attachment) + if scrape_again == False: + print "Not scraping this URL again as requested" + return (None,None) + + time.sleep(3) # wait 3 seconds to give webserver time to recover + #if there is a previous version stored in couchdb, load caching helper tags if doc.has_key('etag'): req.add_header("If-None-Match", doc['etag']) @@ -28,45 +45,60 @@ headers = url_handle.info() # the addinfourls have the .info() too doc['etag'] = headers.getheader("ETag") doc['last_modified'] = headers.getheader("Last-Modified") + doc['date'] = headers.getheader("Date") + doc['page_scraped'] = time.time() doc['web_server'] = headers.getheader("Server") + doc['powered_by'] = headers.getheader("X-Powered-By") doc['file_size'] = headers.getheader("Content-Length") - doc['mime_type'] = headers.getheader("Content-Type") - + doc['mime_type'] = headers.getheader("Content-Type").split(";")[0] if hasattr(url_handle, 'code'): if url_handle.code == 304: print "the web page has not been modified" + return (None,None) else: - #do scraping - html = url_handle.read() - # http://www.crummy.com/software/BeautifulSoup/documentation.html - soup = BeautifulSoup(html) - links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) - for link in links: - if link.has_key("href"): - print link['href'] - #for each unique link - #if html mimetype - # go down X levels, - # diff with last stored attachment, store in document - #if not - # remember to save parentURL and title (link text that lead to document) - + content = url_handle.read() + docsdb.save(doc) + doc = docsdb.get(hash) # need to get a _rev + docsdb.put_attachment(doc, content, str(time.time())+"-"+os.path.basename(url), doc['mime_type']) + return (doc['mime_type'], content) #store as attachment epoch-filename else: - print "error %s in downloading %s", url_handle.code, URL - #record/alert error to error database - - + print "error %s in downloading %s" % url_handle.code, URL + doc['error'] = "error %s in downloading %s" % url_handle.code, URL + docsdb.save(doc) + return (None,None) +def scrapeAndStore(docsdb, url, depth, fieldName, agencyID): + (mime_type,content) = fetchURL(docsdb, url, fieldName, agencyID) + if content != None and depth > 0: + if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": + # http://www.crummy.com/software/BeautifulSoup/documentation.html + soup = BeautifulSoup(content) + navIDs = soup.findAll(id=re.compile('nav|Nav|menu|bar')) + for nav in navIDs: + print "Removing element", nav['id'] + nav.extract() + navClasses = soup.findAll(attrs={'class' : re.compile('nav|menu|bar')}) + for nav in navClasses: + print "Removing element", nav['class'] + nav.extract() + links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) + linkurls = set([]) + for link in links: + if link.has_key("href"): + if link['href'].startswith("http"): + # lets not do external links for now + # linkurls.add(link['href']) + None + else: + linkurls.add(urljoin(url,link['href'].replace(" ","%20"))) + for linkurl in linkurls: + #print linkurl + scrapeAndStore(docsdb, linkurl, depth-1, fieldName, agencyID) - - - - - -couch = couchdb.Server('http://192.168.1.148:5984/') +couch = couchdb.Server('http://127.0.0.1:5984/') # select database agencydb = couch['disclosr-agencies'] @@ -75,5 +107,10 @@ for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view? agency = agencydb.get(row.id) print agency['name'] - scrapeAndStore(docsdb, agency['website'],1,agency['_id']) + for key in agency.keys(): + if key == 'website' or key.endswith('URL'): + print key + scrapeAndStore(docsdb, agency[key],agency['scrapeDepth'],key,agency['_id']) + agency['metadata']['lastscraped'] = time.time() + agencydb.save(agency)