more scrapers, fix column order, extract title from lengthy description
[disclosr.git] / documents / scrapers / 38ca99d2790975a40dde3fae41dbdc3d.py
blob:a/documents/scrapers/38ca99d2790975a40dde3fae41dbdc3d.py -> blob:b/documents/scrapers/38ca99d2790975a40dde3fae41dbdc3d.py
  import sys,os
  sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
  import genericScrapers
  import scrape
  from bs4 import BeautifulSoup
  import dateutil
  from dateutil.parser import *
  from datetime import *
   
  #http://www.doughellmann.com/PyMOTW/abc/
  class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
  def getColumnCount(self):
  return 3
  def getColumns(self,columns):
  (date, title, description) = columns
  return (date, date, title, description, None)
  def getTitle(self, content, entry, doc):
  i = 0
  title = ""
  for string in content.stripped_strings:
  if i < 2:
  title = title + string
  i = i+1
  doc.update({'title': title})
  print title
  return
   
  if __name__ == '__main__':
  print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
  print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
  ScraperImplementation().doScrape()