beginnings of document scrapers mark 2
[disclosr.git] / documents / scrapers / e2a845e55bc9986e6c75c5ad2c508b8d.py
blob:a/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py -> blob:b/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
--- a/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
+++ b/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
@@ -1,1 +1,54 @@
-www.finance.gov.au/foi/disclosure-log/foi-rss.xml
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+import dateutil
+from dateutil.parser import *
+from datetime import *
+import scrape
+from bs4 import BeautifulSoup
+class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
+
+    def __init__(self):
+        super(ScraperImplementation, self).__init__()
+    def getTable(self, soup):
+        return soup.find(id='content')
+
+    def getDescription(self,content, entry,doc):
+        link = None
+        links = []
+        description = ""
+        for atag in entry.find_all('a'):
+            if atag.has_attr('href'):
+                link = scrape.fullurl(self.getURL(), atag['href'])
+                (url, mime_type, htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
+                if htcontent != None:
+                    if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+                        soup = BeautifulSoup(htcontent)
+                        row  = soup.find(id="foidetails")
+			if row == None:
+				row  = soup.find(id="content").table
+			if row == None:
+				row  = soup.find(id="content")
+                        description = ''.join(row.stripped_strings)
+                        for atag in row.find_all("a"):
+                                    if atag.has_attr('href'):
+                                        links.append(scrape.fullurl(link, atag['href']))
+
+        if links != []:
+                     doc.update({'links': links})
+        if description != "":
+            doc.update({ 'description': description})
+
+    def getColumnCount(self):
+        return 3
+
+    def getColumns(self, columns):
+        (id, title, date) = columns
+        return (id, date, title, title, None)
+
+
+if __name__ == '__main__':
+    print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
+    print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
+    ScraperImplementation().doScrape()
+