Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
Former-commit-id: fb652de2c04eb0dcdcc435c47141e3a0034e996b
--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -9,7 +9,6 @@
$obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
-$obj->views->byDate->reduce = "_count";
$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";
$obj->views->byDateMonthYear->reduce = "_count";
$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
--- /dev/null
+++ b/documents/about.php
@@ -1,1 +1,11 @@
+<?php
+include('template.inc.php');
+include_header_documents("");
+include_once('../include/common.inc.php');
+?>
+<h1>About</h1>
+<?php
+include_footer_documents();
+?>
+
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -137,7 +137,12 @@
self.getDescription(description,row, doc)
if notes != None:
doc.update({ 'notes': (''.join(notes.stripped_strings))})
- foidocsdb.save(doc)
+ badtitles = ['-','Summary of FOI Request','FOI request(in summary form)','Summary of FOI request received by the ASC',
+'Summary of FOI request received by agency/minister','Description of Documents Requested','FOI request','Description of FOI Request','Summary of request','Description','Summary',
+'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of FOI Request',"FOI request",'Results 1 to 67 of 67']
+ if doc['title'] not in badtitles and doc['description'] != '':
+ print "saving"
+ foidocsdb.save(doc)
else:
print "already saved "+hash
--- a/documents/rss.xml.php
+++ b/documents/rss.xml.php
@@ -11,7 +11,7 @@
//Use wrapper functions for common channelelements
$TestFeed->setTitle('Last Modified - All');
$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php');
-$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
+$TestFeed->setDescription('Latest entries');
$TestFeed->setChannelElement('language', 'en-us');
$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));
//Retriving informations from database
@@ -21,7 +21,7 @@
$idtoname[$row->id] = trim($row->value->name);
}
$foidocsdb = $server->get_db('disclosr-foidocuments');
-$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00'), true)->rows;
+$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00', 50), true)->rows;
//print_r($rows);
foreach ($rows as $row) {
//Create an empty FeedItem
@@ -38,3 +38,4 @@
//OK. Everything is done. Now genarate the feed.
$TestFeed->generateFeed();
?>
+
--- /dev/null
+++ b/documents/runScrapers.sh
@@ -1,1 +1,3 @@
+for f in scrapers/*.py; do echo "Processing $f file.."; python $f; done
+
--- a/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
+++ b/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
@@ -21,9 +21,10 @@
for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'):
if row != None:
rowtitle = row.find('th').string
- description = description + "\n" + rowtitle + ": "
+ if rowtitle != None:
+ description = description + "\n" + rowtitle + ": "
for text in row.find('td').stripped_strings:
- description = description + text
+ description = description + text
for atag in row.find_all("a"):
if atag.has_key('href'):
links.append(scrape.fullurl(link,atag['href']))
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -21,7 +21,7 @@
<!-- Mobile viewport optimized: h5bp.com/viewport -->
<meta name="viewport" content="width=device-width">
-
+<link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />
<!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />
@@ -89,7 +89,7 @@
<hr>
<footer>
- <p>© Company 2012</p>
+ <p>Not affiliated with or endorsed by any government agency.</p>
</footer>
<script type="text/javascript">
@@ -128,10 +128,25 @@
</html>
<?php
}
-
+function truncate($string, $length, $stopanywhere=false) {
+ //truncates a string to a certain char length, stopping on a word if not specified otherwise.
+ if (strlen($string) > $length) {
+ //limit hit!
+ $string = substr($string,0,($length -3));
+ if ($stopanywhere) {
+ //stop anywhere
+ $string .= '...';
+ } else{
+ //stop on a word.
+ $string = substr($string,0,strrpos($string,' ')).'...';
+ }
+ }
+ return $string;
+}
function displayLogEntry($row, $idtoname) {
$result = "";
- $result .= "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".str_replace("\n","<br>",$row->value->description);
+ $result .= "<div><h2>".$row->value->date.": ".truncate($row->value->title, 80)." (".$idtoname[$row->value->agencyID].")</h2>";
+ echo "<p> Title".$row->value->title."<br/>".str_replace("\n","<br>",$row->value->description);
if (isset($row->value->notes)) {
$result .= " <br>Note: ".$row->value->notes;
}