From: Maxious <maxious@lambdacomplex.org>
Date: Mon, 03 Dec 2012 02:19:28 +0000
Subject: ignore bad titles
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=6a4fe71f9a7a80ac604f7355dd626922979cb69a
---
ignore bad titles


Former-commit-id: f6cbe94c022babb17372fd12848ded4dee2f0f5c
---


--- /dev/null
+++ b/documents/about.php
@@ -1,1 +1,11 @@
+<?php
 
+include('template.inc.php');
+include_header_documents("");
+include_once('../include/common.inc.php');
+?>
+<h1>About</h1>
+<?php
+include_footer_documents();
+?>
+

--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -137,7 +137,12 @@
 							self.getDescription(description,row, doc)
 							if notes != None:
                                         			doc.update({ 'notes': (''.join(notes.stripped_strings))})
-							foidocsdb.save(doc)
+                                                        badtitles = ['-','Summary of FOI Request','FOI request(in summary form)','Summary of FOI request received by the ASC',
+'Summary of FOI request received by agency/minister','Description of Documents Requested','FOI request','Description of FOI Request','Summary of request','Description','Summary',
+'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of    FOI Request',"FOI request",'Results 1 to 67 of 67']
+							if doc['title'] not in badtitles and doc['description'] != '':
+                                                            print "saving"
+                                                            foidocsdb.save(doc)
 						else:
 							print "already saved "+hash
 					

--- a/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
+++ b/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
@@ -21,9 +21,10 @@
                                                 for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'):
                                                         if row != None:
 								rowtitle = row.find('th').string
-                                                                description = description + "\n" + rowtitle + ": "
+                                                                if rowtitle != None:
+                                                                    description = description + "\n" + rowtitle + ": "
                                                                 for text in row.find('td').stripped_strings:
-                                                                        description = description + text
+                                                                    description = description + text
                                                      		for atag in row.find_all("a"):
                                                                 	if atag.has_key('href'):
                                                                         	links.append(scrape.fullurl(link,atag['href']))

--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -89,7 +89,7 @@
       <hr>
 
       <footer>
-        <p>&copy; Company 2012</p>
+        <p>Not affiliated with or endorsed by any government agency.</p>
       </footer>
       <script type="text/javascript">
 
@@ -128,10 +128,25 @@
 </html>
 <?php
 }
-
+function truncate($string, $length, $stopanywhere=false) {
+    //truncates a string to a certain char length, stopping on a word if not specified otherwise.
+    if (strlen($string) > $length) {
+        //limit hit!
+        $string = substr($string,0,($length -3));
+        if ($stopanywhere) {
+            //stop anywhere
+            $string .= '...';
+        } else{
+            //stop on a word.
+            $string = substr($string,0,strrpos($string,' ')).'...';
+        }
+    }
+    return $string;
+}
 function displayLogEntry($row, $idtoname) {
     $result = "";
-    $result .= "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".str_replace("\n","<br>",$row->value->description);
+    $result .= "<div><h2>".$row->value->date.": ".truncate($row->value->title, 80)." (".$idtoname[$row->value->agencyID].")</h2>";
+    echo "<p> Title".$row->value->title."<br/>".str_replace("\n","<br>",$row->value->description);
 if (isset($row->value->notes)) {
 $result .= " <br>Note: ".$row->value->notes;
 }