Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr


Former-commit-id: c29a791360b1cd169ea3062bf04c20cb4e7dced8

[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"] [submodule "couchdb/settee"]
path = couchdb/settee path = couchdb/settee
url = https://github.com/inadarei/settee.git url = https://github.com/inadarei/settee.git
[submodule "lib/php-diff"] [submodule "lib/php-diff"]
path = lib/php-diff path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git url = https://github.com/chrisboulton/php-diff.git
[submodule "lib/Requests"] [submodule "lib/Requests"]
path = lib/Requests path = lib/Requests
url = https://github.com/rmccue/Requests.git url = https://github.com/rmccue/Requests.git
[submodule "js/flotr2"] [submodule "js/flotr2"]
path = js/flotr2 path = js/flotr2
url = https://github.com/HumbleSoftware/Flotr2.git url = https://github.com/HumbleSoftware/Flotr2.git
[submodule "lib/phpquery"] [submodule "lib/phpquery"]
path = lib/phpquery path = lib/phpquery
url = https://github.com/TobiaszCudnik/phpquery.git url = https://github.com/TobiaszCudnik/phpquery.git
[submodule "js/sigma"] [submodule "js/sigma"]
path = js/sigma path = js/sigma
url = https://github.com/jacomyal/sigma.js.git url = https://github.com/jacomyal/sigma.js.git
[submodule "js/bubbletree"] [submodule "js/bubbletree"]
path = js/bubbletree path = js/bubbletree
url = https://github.com/okfn/bubbletree.git url = https://github.com/okfn/bubbletree.git
[submodule "lib/querypath"] [submodule "lib/querypath"]
path = lib/querypath path = lib/querypath
url = https://github.com/technosophos/querypath.git url = https://github.com/technosophos/querypath.git
[submodule "lib/amon-php"] [submodule "lib/amon-php"]
path = lib/amon-php path = lib/amon-php
url = https://github.com/martinrusev/amon-php.git url = https://github.com/martinrusev/amon-php.git
  [submodule "documents/lib/parsedatetime"]
  path = documents/lib/parsedatetime
  url = git://github.com/bear/parsedatetime.git
   
<?php <?php
   
require_once '../include/common.inc.php'; require_once '../include/common.inc.php';
  //function createFOIDocumentsDesignDoc() {
   
  $foidb = $server->get_db('disclosr-foidocuments');
  $obj = new stdClass();
  $obj->_id = "_design/" . urlencode("app");
  $obj->language = "javascript";
  $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
  $obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
   
  // allow safe updates (even if slightly slower due to extra: rev-detection check).
  $foidb->save($obj, true);
   
   
  function createDocumentsDesignDoc() {
  /*
  global $db;
  $obj = new stdClass();
  $obj->_id = "_design/" . urlencode("app");
  $obj->language = "javascript";
  $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
  $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
  "views": {
  "web_server": {
  "map": "function(doc) {\n emit(doc.web_server, 1);\n}",
  "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
  },
  "byAgency": {
  "map": "function(doc) {\n emit(doc.agencyID, 1);\n}",
  "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
  },
  "byURL": {
  "map": "function(doc) {\n emit(doc.url, doc);\n}"
  },
  "agency": {
  "map": "function(doc) {\n emit(doc.agencyID, doc);\n}"
  },
  "byWebServer": {
  "map": "function(doc) {\n emit(doc.web_server, doc);\n}"
  },
  "getValidationRequired": {
  "map": "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}"
  }
  } */
  }
   
  //function createAgencyDesignDoc() {
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
createAgencyDesignDoc(); $obj = new stdClass();
  $obj->_id = "_design/" . urlencode("app");
  $obj->language = "javascript";
  $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
  $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
  $obj->views->byCanonicalName->map = "function(doc) {
  if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
  emit(doc.name, doc);
  }
  };";
  $obj->views->byDeptStateName->map = "function(doc) {
  if (doc.orgType == 'FMA-DepartmentOfState') {
  emit(doc.name, doc._id);
  }
  };";
  $obj->views->parentOrgs->map = "function(doc) {
  if (doc.parentOrg) {
  emit(doc._id, doc.parentOrg);
  }
  };";
  $obj->views->byName->map = 'function(doc) {
  if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
  emit(doc.name, doc._id);
  if (typeof(doc.shortName) != "undefined" && doc.shortName != doc.name) {
  emit(doc.shortName, doc._id);
  }
  for (name in doc.otherNames) {
  if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) {
  emit(doc.otherNames[name], doc._id);
  }
  }
  for (name in doc.foiBodies) {
  if (doc.foiBodies[name] != "" && doc.foiBodies[name] != doc.name) {
  emit(doc.foiBodies[name], doc._id);
  }
  }
  }
  };';
   
  $obj->views->foiEmails->map = "function(doc) {
  emit(doc._id, doc.foiEmail);
  };";
   
  $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
  $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
  $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
  $obj->views->getScrapeRequired->map = "function(doc) {
   
  var lastScrape = Date.parse(doc.metadata.lastScraped);
   
  var today = new Date();
   
  if (!lastScrape || lastScrape.getTime() + 1000 != today.getTime()) {
  emit(doc._id, doc);
  }
   
  };";
  $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
  $obj->views->getConflicts->map = "function(doc) {
  if (doc._conflicts) {
  emit(null, [doc._rev].concat(doc._conflicts));
  }
  }";
  // http://stackoverflow.com/questions/646628/javascript-startswith
  $obj->views->score->map = 'if(!String.prototype.startsWith){
  String.prototype.startsWith = function (str) {
  return !this.indexOf(str);
  }
  }
   
  function(doc) {
  count = 0;
  if (doc["status"] != "suspended") {
  for(var propName in doc) {
  if(typeof(doc[propName]) != "undefined" && doc[propName] != "") {
  count++;
  }
  }
  portfolio = doc.parentOrg;
  if (doc.orgType == "FMA-DepartmentOfState") {
  portfolio = doc._id;
  }
  if (doc.orgType == "Court-Commonwealth" || doc.orgType == "FMA-DepartmentOfParliament") {
  portfolio = doc.orgType;
  }
  emit(count+doc._id, {id:doc._id, name: doc.name, score:count, orgType: doc.orgType, portfolio:portfolio});
  }
  }';
  $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
  String.prototype.startsWith = function (str) {
  return !this.indexOf(str);
  }
  }
  if(!String.prototype.endsWith){
  String.prototype.endsWith = function(suffix) {
      return this.indexOf(suffix, this.length - suffix.length) !== -1;
  };
  }
  function(doc) {
  if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
  for(var propName in doc) {
  if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") || propName.endsWith("URL"))) {
  emit(propName, 1);
  }
  }
  emit("total", 1);
  }
  }';
  $obj->views->scoreHas->reduce = 'function (key, values, rereduce) {
  return sum(values);
  }';
  $obj->views->fieldNames->map = '
  function(doc) {
  for(var propName in doc) {
  emit(propName, doc._id);
  }
   
  }';
  $obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
  return values.length;
  }';
  // allow safe updates (even if slightly slower due to extra: rev-detection check).
  $db->save($obj, true);
   
   
?> ?>
   
<?php  
 
include_once('../include/common.inc.php');  
include_header('Webserver and Accessiblity');  
 
echo "<table>  
<tr><th>name</th><th>disclog</th><th>scraper?</th></tr>";  
$agenciesdb = $server->get_db('disclosr-agencies');  
$docsdb = $server->get_db('disclosr-documents');  
try {  
$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;  
 
 
if ($rows) {  
foreach ($rows as $row) {  
 
echo "<tr><td>" . $row->value->name . " (".$row->id.")</td>\n";  
echo "<td>";  
if (isset($row->value->FOIDocumentsURL)) {  
echo '<a href="viewDocument.php?hash='.md5($row->value->FOIDocumentsURL).'">'  
.$row->value->FOIDocumentsURL.'</a>';  
} else {  
echo "<font color='red'>✘</font>";  
}  
echo "</td>\n<td>";  
if (isset($row->value->FOIDocumentsURL)) {  
if (file_exists("./scrapers/".$row->id.'.py')) {  
echo "<font color='green'>✔</font>";  
} else if (file_exists("./scrapers/".$row->id.'.txt')){  
echo "pass";  
} else {  
echo "<font color='red'>✘</font>";  
}  
}  
echo "</td></tr>\n";  
}  
}  
} catch (SetteeRestClientException $e) {  
setteErrorHandler($e);  
}  
include_footer();  
?>  
  /*!
  * Bootstrap Responsive v2.2.1
  *
  * Copyright 2012 Twitter, Inc
  * Licensed under the Apache License v2.0
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Designed and built with all the love in the world @twitter by @mdo and @fat.
  */
 
  .clearfix {
  *zoom: 1;
  }
 
  .clearfix:before,
  .clearfix:after {
  display: table;
  line-height: 0;
  content: "";
  }
 
  .clearfix:after {
  clear: both;
  }
 
  .hide-text {
  font: 0/0 a;
  color: transparent;
  text-shadow: none;
  background-color: transparent;
  border: 0;
  }
 
  .input-block-level {
  display: block;
  width: 100%;
  min-height: 30px;
  -webkit-box-sizing: border-box;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
  }
 
  .hidden {
  display: none;
  visibility: hidden;
  }
 
  .visible-phone {
  display: none !important;
  }
 
  .visible-tablet {
  display: none !important;
  }
 
  .hidden-desktop {
  display: none !important;
  }
 
  .visible-desktop {
  display: inherit !important;
  }
 
  @media (min-width: 768px) and (max-width: 979px) {
  .hidden-desktop {
  display: inherit !important;
  }
  .visible-desktop {
  display: none !important ;
  }
  .visible-tablet {
  display: inherit !important;
  }
  .hidden-tablet {
  display: none !important;
  }
  }
 
  @media (max-width: 767px) {
  .hidden-desktop {
  display: inherit !important;
  }
  .visible-desktop {
  display: none !important;
  }
  .visible-phone {
  display: inherit !important;
  }
  .hidden-phone {
  display: none !important;
  }
  }