more scrapers, multipage scrapers
[disclosr.git] / documents / scrapers / ac6cf1ebaa79694e2cc3705bc83c89b3.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers
import scrape
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import date
 
#http://www.doughellmann.com/PyMOTW/abc/
class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
        def getColumnCount(self):
                return 3
        def getTable(self,soup):
                return soup.find(class_ = "internal")
        def getColumns(self,columns):
                (id, date, description) = columns
                return (id, date, description, description, None)
        def getTitle(self, content, entry, doc):
                doc.update({'title': content.stripped_strings.next()})
                return
        def getDate(self, content, entry, doc):
                edate = datetime.strptime(content.string.strip(), "%d/%m/%Y").strftime("%Y-%m-%d")
                print edate
                doc.update({'date': edate})
                return
 
if __name__ == '__main__':
    print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
    print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
    ScraperImplementation().doScrape()