From: root <root@cloudnine.(none)>
Date: Mon, 03 Dec 2012 02:56:21 +0000
Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=e2b91982ab62f9f22d63338fea8e9f462d4004ca
---
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr


Former-commit-id: fb652de2c04eb0dcdcc435c47141e3a0034e996b
---


--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -9,7 +9,6 @@
 $obj->language = "javascript";
 $obj->views->all->map = "function(doc) {   emit(doc._id, doc); };";
 $obj->views->byDate->map = "function(doc) {   emit(doc.date, doc); };";
-$obj->views->byDate->reduce = "_count";
 $obj->views->byDateMonthYear->map = "function(doc) {   emit(doc.date, doc); };";
 $obj->views->byDateMonthYear->reduce = "_count";
 $obj->views->byAgencyID->map = "function(doc) {   emit(doc.agencyID, doc); };";

--- /dev/null
+++ b/documents/about.php
@@ -1,1 +1,11 @@
+<?php
 
+include('template.inc.php');
+include_header_documents("");
+include_once('../include/common.inc.php');
+?>
+<h1>About</h1>
+<?php
+include_footer_documents();
+?>
+

--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -137,7 +137,12 @@
 							self.getDescription(description,row, doc)
 							if notes != None:
                                         			doc.update({ 'notes': (''.join(notes.stripped_strings))})
-							foidocsdb.save(doc)
+                                                        badtitles = ['-','Summary of FOI Request','FOI request(in summary form)','Summary of FOI request received by the ASC',
+'Summary of FOI request received by agency/minister','Description of Documents Requested','FOI request','Description of FOI Request','Summary of request','Description','Summary',
+'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of    FOI Request',"FOI request",'Results 1 to 67 of 67']
+							if doc['title'] not in badtitles and doc['description'] != '':
+                                                            print "saving"
+                                                            foidocsdb.save(doc)
 						else:
 							print "already saved "+hash
 					

--- a/documents/rss.xml.php
+++ b/documents/rss.xml.php
@@ -11,7 +11,7 @@
 //Use wrapper functions for common channelelements
 $TestFeed->setTitle('Last Modified - All');
 $TestFeed->setLink('http://disclosurelo.gs/rss.xml.php');
-$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
+$TestFeed->setDescription('Latest entries');
   $TestFeed->setChannelElement('language', 'en-us');
   $TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));
 //Retriving informations from database
@@ -21,7 +21,7 @@
     $idtoname[$row->id] = trim($row->value->name);
 }
 $foidocsdb = $server->get_db('disclosr-foidocuments');
-$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00'), true)->rows;
+$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00', 50), true)->rows;
 //print_r($rows);
 foreach ($rows as $row) {
     //Create an empty FeedItem
@@ -38,3 +38,4 @@
 //OK. Everything is done. Now genarate the feed.
 $TestFeed->generateFeed();
 ?>
+

--- /dev/null
+++ b/documents/runScrapers.sh
@@ -1,1 +1,3 @@
+for f in scrapers/*.py; do echo "Processing $f file.."; python $f; done
 
+

--- a/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
+++ b/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
@@ -21,9 +21,10 @@
                                                 for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'):
                                                         if row != None:
 								rowtitle = row.find('th').string
-                                                                description = description + "\n" + rowtitle + ": "
+                                                                if rowtitle != None:
+                                                                    description = description + "\n" + rowtitle + ": "
                                                                 for text in row.find('td').stripped_strings:
-                                                                        description = description + text
+                                                                    description = description + text
                                                      		for atag in row.find_all("a"):
                                                                 	if atag.has_key('href'):
                                                                         	links.append(scrape.fullurl(link,atag['href']))

--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -21,7 +21,7 @@
 
   <!-- Mobile viewport optimized: h5bp.com/viewport -->
   <meta name="viewport" content="width=device-width">
-
+<link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />
   <!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
 <meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />
 
@@ -89,7 +89,7 @@
       <hr>
 
       <footer>
-        <p>&copy; Company 2012</p>
+        <p>Not affiliated with or endorsed by any government agency.</p>
       </footer>
       <script type="text/javascript">
 
@@ -128,10 +128,25 @@
 </html>
 <?php
 }
-
+function truncate($string, $length, $stopanywhere=false) {
+    //truncates a string to a certain char length, stopping on a word if not specified otherwise.
+    if (strlen($string) > $length) {
+        //limit hit!
+        $string = substr($string,0,($length -3));
+        if ($stopanywhere) {
+            //stop anywhere
+            $string .= '...';
+        } else{
+            //stop on a word.
+            $string = substr($string,0,strrpos($string,' ')).'...';
+        }
+    }
+    return $string;
+}
 function displayLogEntry($row, $idtoname) {
     $result = "";
-    $result .= "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".str_replace("\n","<br>",$row->value->description);
+    $result .= "<div><h2>".$row->value->date.": ".truncate($row->value->title, 80)." (".$idtoname[$row->value->agencyID].")</h2>";
+    echo "<p> Title".$row->value->title."<br/>".str_replace("\n","<br>",$row->value->description);
 if (isset($row->value->notes)) {
 $result .= " <br>Note: ".$row->value->notes;
 }