scrapers work
[disclosr.git] / documents / scrapers / be9996f0ac58f71f23d074e82d44ead3.py
blob:a/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py -> blob:b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
--- a/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
+++ b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
@@ -1,11 +1,42 @@
 import sys,os
 sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+#RSS feed not detailed
 import scrape
-foidocsdb = scrape.couch['disclosr-foidocuments']
-
-import feedparser
-feed = feedparser.parse( "http://foi.deewr.gov.au/disclosure-log/rss")
-print feed.entries[0]
-#foreach feed.entries
+from bs4 import BeautifulSoup
+#http://www.doughellmann.com/PyMOTW/abc/
+class ScraperImplementation(genericScrapers.GenericRSSDisclogScraper):
+	def getDescription(self,content, entry,doc):
+		(url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, entry.link, "foidocuments", self.getAgencyID(), False)
+                if htcontent != None:
+                        if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+                        # http://www.crummy.com/software/BeautifulSoup/documentation.html
+                               	soup = BeautifulSoup(content)
+				links = []
+				description = ""
+				dldivs = soup.find('div',class_="download")
+				if dldivs != None:
+                             		for atag in dldivs.find_all("a"):
+                                		if atag.has_attr('href'):
+                                        		links.append(scrape.fullurl(url,atag['href']))
+				nodldivs = soup.find('div',class_="incompleteNotification")
+				if nodldivs != None and nodldivs.stripped_strings != None:
+                                      	for text in nodldivs.stripped_strings:
+	                                        description = description + text
+                               	for row in soup.table.find_all('tr'):
+					if row != None:
+						description = description + "\n" + row.find('th').string + ": " 
+						for text in row.find('div').stripped_strings:
+							 description = description + text
+				if links != []:
+					doc.update({'links': links})
+				if description != "":
+					doc.update({ 'description': description})
 
 
+if __name__ == '__main__':
+    print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericRSSDisclogScraper)
+    print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericRSSDisclogScraper)
+    ScraperImplementation().doScrape()
+
+