From: Alex Sadleir <maxious@lambdacomplex.org>
Date: Sun, 03 Nov 2013 06:20:13 +0000
Subject: sort agencies by name
X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=a377d3925bd3611b7507e3b51a900f4d4274cf5d
---
sort agencies by name


Former-commit-id: 24f1070892cb642be8833a65922d273623957f27
---


--- /dev/null
+++ b/admin/massdelete.sh
@@ -1,1 +1,10 @@
+for line in `curl "http://localhost:5984/disclosr-foidocuments/_design/app/_view/byAgencyID?reduce=false&keys=%5B\"5716ce0aacfe98f7d638b7a66b7f1040\"%5D&limit=600" | xargs -L1`; do
+#	echo $line
+	id=`echo $line | grep -Po '_id:.*?[^\\\],' | perl -pe 's/_id://; s/^//; s/,$//'`
+	rev=`echo $line | grep -Po 'rev:.*?[^\\\],'| perl -pe 's/rev://; s/^//; s/,$//'`
+	if [ -n "$id" ]; then
+		echo "curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev"
+		curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev
+	fi
+done;
 

--- a/documents/about.php
+++ b/documents/about.php
@@ -5,6 +5,7 @@
 include_once('../include/common.inc.php');
 ?>
 <h1>About</h1>
+Written and managed by Alex Sadleir (maxious [at] lambdacomplex.org) 
 <?php
 include_footer_documents();
 ?>

--- a/documents/agency.php
+++ b/documents/agency.php
@@ -31,6 +31,12 @@
     } else {
         $rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows;
         if ($rows) {
+function cmp($a, $b)
+{
+	global $idtoname;
+    return strcmp($idtoname[$a->key], $idtoname[$b->key]);
+}
+usort($rows, "cmp");
             foreach ($rows as $row) {
                 echo '<a href="agency.php?id=' . $row->key . '">' . $idtoname[$row->key] . " (" . $row->value . " records)</a> <br>\n";
             }

--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -199,6 +199,17 @@
     def getRows(self, table):
         return table.find_all('tr')
 
+    def getDocHash(self, id,date, url):
+                        if id.string is None:
+			    print "no id, using date as hash"
+                            return scrape.mkhash(
+                                self.remove_control_chars(
+                                    url + (''.join(date.stripped_strings))))
+                        else:
+                            return scrape.mkhash(
+                                self.remove_control_chars(
+                                    url + (''.join(id.stripped_strings))))
+
     def getDate(self, content, entry, doc):
         strdate = ''.join(content.stripped_strings).strip()
         (a, b, c) = strdate.partition("(")
@@ -240,15 +251,7 @@
                          description, notes) = self.getColumns(columns)
                         print self.remove_control_chars(
                             ''.join(id.stripped_strings))
-                        if id.string is None:
-			    print "no id, using date as hash"
-                            dochash = scrape.mkhash(
-                                self.remove_control_chars(
-                                    url + (''.join(date.stripped_strings))))
-                        else:
-                            dochash = scrape.mkhash(
-                                self.remove_control_chars(
-                                    url + (''.join(id.stripped_strings))))
+                        dochash = self.getDocHash(id,date,url)
                         doc = foidocsdb.get(dochash)
 
                         if doc is None:

--- a/documents/runScrapers.sh
+++ b/documents/runScrapers.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 echo $DIR
 cd $DIR

--- a/documents/scrapers/5716ce0aacfe98f7d638b7a66b7f1040.py
+++ b/documents/scrapers/5716ce0aacfe98f7d638b7a66b7f1040.py
@@ -6,6 +6,11 @@
 
 #http://www.doughellmann.com/PyMOTW/abc/
 class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
+        def getDocHash(self, id,date, url):
+		''' url changes on ever request so ignore for hash '''
+		return scrape.mkhash(
+                                self.remove_control_chars(
+                                    ''.join(id.stripped_strings)))
         def getColumnCount(self):
                 return 4
         def getColumns(self,columns):

--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -77,7 +77,7 @@
                     </p>
                     <ul class="nav">
                         <li><a href="agency.php">By Agency</a></li>
-                        <li><a href="date.php">By Date</a></li>
+<!--                        <li><a href="date.php">By Date</a></li> -->
                         <li><a href="disclogsList.php">List of Disclosure Logs</a></li>
                         <li><a href="charts.php">Charts</a></li>
                         <li><a href="about.php">About</a></li>

--- a/graph.php
+++ b/graph.php
@@ -9,13 +9,13 @@
 function add_node($id, $label, $parent="") {
     global $format;
     if ($format == "html") {
-       // echo "nodes[\"$id\"] = graph.newNode({label: \"$label\"});" . PHP_EOL;
+      //  echo "nodes[\"$id\"] = graph.newNode({label: \"$label\"});" . PHP_EOL;
     }
      if ($format == "dot" && $label != "") {
-         echo "$id [label=\"$label\"];". PHP_EOL;
+         echo "\"$id\" [label=\"$label\", shape=plaintext];". PHP_EOL;
      }
       if ($format == "gexf") {
-          echo "<node id='$id' label=\"".htmlentities($label,ENT_XML1)."\" ".($parent != ""? "pid='$parent'><viz:size value='1'/>":"><viz:size value='2'/>")
+          echo "<node id='$id' label=\"".htmlentities($label)."\" ".($parent != ""? "pid='$parent'><viz:size value='1'/>":"><viz:size value='2'/>")
               ."<viz:color b='".rand(0,255)."' g='".rand(0,255)."' r='".rand(0,255)."'/>"
                   ."</node>". PHP_EOL;
       }
@@ -27,7 +27,7 @@
      //   echo "graph.newEdge(nodes[\"$from\"], nodes['$to'], {color: '$color'});" . PHP_EOL;
     }
     if ($format == "dot") {
-        echo "$from -> $to ".($color != ""? "[color=$color]":"").";". PHP_EOL;
+        echo "\"$from\" -> \"$to\" ".($color != ""? "[color=$color]":"").";". PHP_EOL;
     }
      if ($format == "gexf") {
           echo "<edge id='$from$to' source='$from' target='$to' />". PHP_EOL;
@@ -55,7 +55,7 @@
     $rows = $db->get_view("app", "byCanonicalName", null, true)->rows;
 //print_r($rows);
     foreach ($rows as $row) {
-        add_node($row->id, $row->key);
+        add_node($row->id, $row->value->name);
     }
 } catch (SetteeRestClientException $e) {
     setteErrorHandler($e);

--- a/ranking.php
+++ b/ranking.php
@@ -32,8 +32,12 @@
                 $columnKeys = array_unique(array_merge($columnKeys, array_keys($columns)));
                 //print_r($columnKeys);
                 $score = count($columns);
-                $scores[$score]++;
-                $scoredagencies[] = Array("id"=> $row->key, "website"=> $row->value->website, "name" => $row->value->name, "columns" => $columns, "score" => $score);
+                if (isset($scores[$score])){
+$scores[$score]++;
+} else {
+$scores[$score] =1;
+}
+                $scoredagencies[] = Array("id"=> $row->key, "website"=> (isset($row->value->website)?$row->value->website:""), "name" => $row->value->name, "columns" => $columns, "score" => $score);
             }
         }
 
@@ -74,7 +78,7 @@
                 } else {
                     $href = $value;
                 }
-                if ($href[0] == "@") {
+                if (isset($href[0]) && $href[0] == "@") {
                     $href = str_replace("@","https://twitter.com/",$href);
                 }
                 //$href= urlencode($href);