--- a/scrape.py +++ b/scrape.py @@ -4,9 +4,7 @@ from BeautifulSoup import BeautifulSoup import re -couch = couchdb.Server() # Assuming localhost:5984 -# If your CouchDB server is running elsewhere, set it up like this: -# couch = couchdb.Server('http://example.com:5984/') +couch = couchdb.Server('http://192.168.1.148:5984/') # select database agencydb = couch['disclosr-agencies'] @@ -41,24 +39,27 @@ file_size = headers.getheader("Content-Length") mime_type = headers.getheader("Content-Type") - if hasattr(url_handle, 'code') and url_handle.code == 304: - print "the web page has not been modified" + if hasattr(url_handle, 'code') + if url_handle.code == 304: + print "the web page has not been modified" + else: + #do scraping + html = url_handle.read() + # http://www.crummy.com/software/BeautifulSoup/documentation.html + soup = BeautifulSoup(html) + links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) + for link in links: + print link['href'] + #for each unique link + #if html mimetype + # go down X levels, + # diff with last stored attachment, store in document + #if not + # remember to save parentURL and title (link text that lead to document) + + #store as attachment epoch-filename else: print "error %s in downloading %s", url_handle.code, URL #record/alert error to error database - #do scraping - html = ? - # http://www.crummy.com/software/BeautifulSoup/documentation.html - soup = BeautifulSoup(html) -links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) -for link in links: - print link['href'] - #for each unique link - #if html mimetype - # go down X levels, - # diff with last stored attachment, store in document - #if not - # remember to save parentURL and title (link text that lead to document) - #store as attachment epoch-filename