more scrapers, fix column order, extract title from lengthy description
[disclosr.git] / documents / scrapers / 795c33ed030293dbdb155c909ea47e37.py
blob:a/documents/scrapers/795c33ed030293dbdb155c909ea47e37.py -> blob:b/documents/scrapers/795c33ed030293dbdb155c909ea47e37.py
import sys,os import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers import genericScrapers
import scrape import scrape
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
   
#http://www.doughellmann.com/PyMOTW/abc/ #http://www.doughellmann.com/PyMOTW/abc/
class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
def getColumnCount(self): def getColumnCount(self):
return 7 return 7
def getColumns(self,columns): def getColumns(self,columns):
(id, date, title, description, notes, deletedate, otherinfo) = columns (id, date, title, description, notes, deletedate, otherinfo) = columns
return (id, date, description, title, notes) return (id, date, title, description, notes)
#def getTable(self,soup): #def getTable(self,soup):
# return soup.find(class_ = "box").table # return soup.find(class_ = "box").table
   
if __name__ == '__main__': if __name__ == '__main__':
print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
ScraperImplementation().doScrape() ScraperImplementation().doScrape()