From: maxious Date: Sat, 04 Feb 2012 11:59:20 +0000 Subject: Don't hit the same URL twice in the same day X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=d54e7489c2429c31e23118a4661d1e102af66160 --- Don't hit the same URL twice in the same day Former-commit-id: fc7d691cec408a85b38dc74e1fece1e3f10f388e --- --- a/charts.php +++ b/charts.php @@ -41,6 +41,8 @@ ?> $.plot($("#placeholder"), [ d1], { + grid: { hoverable: true }, + series: { bars: { show: true, barWidth: 0.6 } }, @@ -58,11 +60,41 @@ } }); +var previousPoint = null; +$("#placeholder").bind("plothover", function (event, pos, item) { + if (item) { + if (previousPoint != item.datapoint) { + previousPoint = item.datapoint; + + $("#tooltip").remove(); + var x = item.datapoint[0], + y = item.datapoint[1] - item.datapoint[2]; + + showTooltip(item.pageX, item.pageY, y ); + } + } + else { + $("#tooltip").remove(); + previousPoint = null; + } +}); }); }; + function showTooltip(x, y, contents) { + $('
' + contents + '
').css( { + position: 'absolute', + display: 'none', + top: y + 5, + left: x + 5, + border: '1px solid #fdd', + padding: '2px', + 'background-color': '#fee', + opacity: 0.80 + }).appendTo("body").fadeIn(200); + } views->byName->map = "function(doc) { + $obj->views->byName->map = 'function(doc) { + if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { emit(doc.name, doc._id); for (name in doc.otherNames) { -if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { +if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) { emit(doc.otherNames[name], doc._id); } } -};"; + } +};'; $obj->views->foiEmails->map = "function(doc) { emit(doc._id, doc.foiEmail); --- a/schemas/agency.json.php +++ b/schemas/agency.json.php @@ -24,8 +24,10 @@ "consultanciesURL" => Array("type" => "string", "required" => true, "x-title" => "Consultants Hired", "description" => ""), "legalExpenditureURL" => Array("type" => "string", "required" => true, "x-title" => "Legal Services Expenditure", "description" => "Legal Services Expenditure mandated by Legal Services Directions 2005"), "recordsListURL" => Array("type" => "string", "required" => true, "x-title" => "Files/Records Held", "description" => "Indexed lists of departmental and agency files, mandated by the Senate"), - "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => ""), - "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), + "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => "FOI Disclosure Log URL"), + "FOIDocumentsRSSURL" => Array("type" => "string", "required" => false, "x-title" => "RSS Feed of FOI Documents Released", "description" => "FOI Disclosure Log in RSS format"), + "hasFOIPDF" => Array("type" => "string", "required" => false, "x-title" => "Has FOI Documents Released in PDF", "description" => "FOI Disclosure Log contains any PDFs"), + "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), "appointmentsURL" => Array("type" => "string", "required" => true, "x-title" => "Agency Appointments/Boards", "description" => "Departmental and agency appointments and vacancies , mandated by the Senate"), "advertisingURL" => Array("type" => "string", "required" => true, "x-title" => "Approved Advertising Campaigns", "description" => " Agency advertising and public information projects, mandated by the Senate "), "hasRSS" => Array("type" => "string", "required" => true, "x-title" => "Has RSS", "description" => ""), --- a/scrape.py +++ b/scrape.py @@ -3,6 +3,10 @@ import urllib2 from BeautifulSoup import BeautifulSoup import re +import hashlib +from urlparse import urljoin +import time +import os #http://diveintopython.org/http_web_services/etags.html class NotModifiedHandler(urllib2.BaseHandler): @@ -11,66 +15,102 @@ addinfourl.code = code return addinfourl -def scrapeAndStore(URL, depth, agency): - URL = "http://www.google.com" - req = urllib2.Request(URL) - etag = 'y' - last_modified = 'y' - #if there is a previous version sotred in couchdb, load caching helper tags - if etag: - req.add_header("If-None-Match", etag) - if last_modified: - req.add_header("If-Modified-Since", last_modified) +def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True): + hash = hashlib.md5(url).hexdigest() + req = urllib2.Request(url) + print "Fetching %s" % url + doc = docsdb.get(hash) + if doc == None: + doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName} + else: + if (time.time() - doc['page_scraped']) < 3600: + print "Uh oh, trying to scrape URL again too soon!" + last_attachment_fname = doc["_attachments"].keys()[-1] + last_attachment = docsdb.get_attachment(doc,last_attachment_fname) + return (doc['mime_type'],last_attachment) + if scrape_again == False: + print "Not scraping this URL again as requested" + return (None,None) + + time.sleep(3) # wait 3 seconds to give webserver time to recover + + #if there is a previous version stored in couchdb, load caching helper tags + if doc.has_key('etag'): + req.add_header("If-None-Match", doc['etag']) + if doc.has_key('last_modified'): + req.add_header("If-Modified-Since", doc['last_modified']) opener = urllib2.build_opener(NotModifiedHandler()) url_handle = opener.open(req) headers = url_handle.info() # the addinfourls have the .info() too - etag = headers.getheader("ETag") - last_modified = headers.getheader("Last-Modified") - web_server = headers.getheader("Server") - file_size = headers.getheader("Content-Length") - mime_type = headers.getheader("Content-Type") - + doc['etag'] = headers.getheader("ETag") + doc['last_modified'] = headers.getheader("Last-Modified") + doc['date'] = headers.getheader("Date") + doc['page_scraped'] = time.time() + doc['web_server'] = headers.getheader("Server") + doc['powered_by'] = headers.getheader("X-Powered-By") + doc['file_size'] = headers.getheader("Content-Length") + doc['mime_type'] = headers.getheader("Content-Type").split(";")[0] if hasattr(url_handle, 'code'): if url_handle.code == 304: print "the web page has not been modified" + return (None,None) else: - #do scraping - html = url_handle.read() - # http://www.crummy.com/software/BeautifulSoup/documentation.html - soup = BeautifulSoup(html) - links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) - for link in links: - print link['href'] - #for each unique link - #if html mimetype - # go down X levels, - # diff with last stored attachment, store in document - #if not - # remember to save parentURL and title (link text that lead to document) - + content = url_handle.read() + docsdb.save(doc) + doc = docsdb.get(hash) # need to get a _rev + docsdb.put_attachment(doc, content, str(time.time())+"-"+os.path.basename(url), doc['mime_type']) + return (doc['mime_type'], content) #store as attachment epoch-filename else: - print "error %s in downloading %s", url_handle.code, URL - #record/alert error to error database - - + print "error %s in downloading %s" % url_handle.code, URL + doc['error'] = "error %s in downloading %s" % url_handle.code, URL + docsdb.save(doc) + return (None,None) +def scrapeAndStore(docsdb, url, depth, fieldName, agencyID): + (mime_type,content) = fetchURL(docsdb, url, fieldName, agencyID) + if content != None and depth > 0: + if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": + # http://www.crummy.com/software/BeautifulSoup/documentation.html + soup = BeautifulSoup(content) + navIDs = soup.findAll(id=re.compile('nav|Nav|menu|bar')) + for nav in navIDs: + print "Removing element", nav['id'] + nav.extract() + navClasses = soup.findAll(attrs={'class' : re.compile('nav|menu|bar')}) + for nav in navClasses: + print "Removing element", nav['class'] + nav.extract() + links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) + linkurls = set([]) + for link in links: + if link.has_key("href"): + if link['href'].startswith("http"): + # lets not do external links for now + # linkurls.add(link['href']) + None + else: + linkurls.add(urljoin(url,link['href'].replace(" ","%20"))) + for linkurl in linkurls: + #print linkurl + scrapeAndStore(docsdb, linkurl, depth-1, fieldName, agencyID) - - - - - -couch = couchdb.Server('http://192.168.1.148:5984/') +couch = couchdb.Server('http://127.0.0.1:5984/') # select database agencydb = couch['disclosr-agencies'] +docsdb = couch['disclosr-documents'] for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view? agency = agencydb.get(row.id) print agency['name'] -scrapeAndStore("A",1,1) + for key in agency.keys(): + if key == 'website' or key.endswith('URL'): + print key + scrapeAndStore(docsdb, agency[key],agency['scrapeDepth'],key,agency['_id']) + agency['metadata']['lastscraped'] = time.time() + agencydb.save(agency)