--- a/documents/genericScrapers.py +++ b/documents/genericScrapers.py @@ -4,6 +4,7 @@ from bs4 import BeautifulSoup import abc +import dateutil.parser class GenericOAICDisclogScraper(object): __metaclass__ = abc.ABCMeta @@ -46,8 +47,9 @@ if doc == None: print "saving" + edate = dateutil.parser.parse(date.string).date().strftime("%Y-%m-%d") doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), "links": links, 'docID': id.string, - "date": date.string, "description": descriptiontxt,"title": title.string,"notes": notes.string} + "date": edate, "description": descriptiontxt,"title": title.string,"notes": notes.string} foidocsdb.save(doc) else: print "already saved"