scraoer fxies
[disclosr.git] / documents / genericScrapers.py
blob:a/documents/genericScrapers.py -> blob:b/documents/genericScrapers.py
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -199,10 +199,21 @@
     def getRows(self, table):
         return table.find_all('tr')
 
+    def getDocHash(self, id,date, url):
+                        if id.string is None:
+			    print "no id, using date as hash"
+                            return scrape.mkhash(
+                                self.remove_control_chars(
+                                    url + (''.join(date.stripped_strings))))
+                        else:
+                            return scrape.mkhash(
+                                self.remove_control_chars(
+                                    url + (''.join(id.stripped_strings))))
+
     def getDate(self, content, entry, doc):
         strdate = ''.join(content.stripped_strings).strip()
         (a, b, c) = strdate.partition("(")
-        strdate = self.remove_control_chars(a.replace("Octber", "October").replace("Janrurary", "January").replace("1012","2012"))
+        strdate = self.remove_control_chars(a.replace("Octber", "October").replace("1012","2012").replace("Janrurary", "January").replace("1012","2012"))
         print strdate
         try:
 		edate = parse(strdate, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
@@ -240,14 +251,7 @@
                          description, notes) = self.getColumns(columns)
                         print self.remove_control_chars(
                             ''.join(id.stripped_strings))
-                        if id.string is None:
-                            dochash = scrape.mkhash(
-                                self.remove_control_chars(
-                                    url + (''.join(date.stripped_strings))))
-                        else:
-                            dochash = scrape.mkhash(
-                                self.remove_control_chars(
-                                    url + (''.join(id.stripped_strings))))
+                        dochash = self.getDocHash(id,date,url)
                         doc = foidocsdb.get(dochash)
 
                         if doc is None: