projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr


Former-commit-id: bbccb1832249b9cf4cc1e15556a2b7cafde596da

file:a/.gitmodules -> file:b/.gitmodules
[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"] [submodule "couchdb/settee"]
path = couchdb/settee path = couchdb/settee
url = https://github.com/inadarei/settee.git url = https://github.com/inadarei/settee.git
[submodule "lib/springy"] [submodule "lib/springy"]
path = lib/springy path = lib/springy
url = https://github.com/dhotson/springy.git url = https://github.com/dhotson/springy.git
  [submodule "lib/php-diff"]
  path = lib/php-diff
  url = https://github.com/chrisboulton/php-diff.git
   
file:a/admin/import.php -> file:b/admin/import.php
<?php <?php
   
require_once 'include/common.inc.php'; require_once '../include/common.inc.php';
try { try {
$server->create_db('disclosr-agencies'); $server->create_db('disclosr-agencies');
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
createAgencyDesignDoc(); createAgencyDesignDoc();
$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost"); $conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");
$namesQ = 'select agency.abn, string_agg("agencyName",\'|\') as names from agency inner join agency_nametoabn on agency.abn::text = agency_nametoabn.abn group by agency.abn;'; $namesQ = 'select agency.abn, string_agg("agencyName",\'|\') as names from agency inner join agency_nametoabn on agency.abn::text = agency_nametoabn.abn group by agency.abn;';
$abntonames = Array(); $abntonames = Array();
foreach ($conn->query($namesQ) as $row) { foreach ($conn->query($namesQ) as $row) {
$abntonames[$row['abn']] = explode("|", $row['names']); $abntonames[$row['abn']] = explode("|", $row['names']);
} }
$result = $conn->query("select * from agency"); $result = $conn->query("select * from agency");
while ($agency = $result->fetch(PDO::FETCH_ASSOC)) { while ($agency = $result->fetch(PDO::FETCH_ASSOC)) {
$agency['_id'] = md5($agency['abn']); $agency['_id'] = md5($agency['abn']);
$agency['otherNames'] = $abntonames[$agency['abn']]; $agency['otherNames'] = $abntonames[$agency['abn']];
if (sizeof($abntonames[$agency['abn']]) == 1) if (sizeof($abntonames[$agency['abn']]) == 1)
$agency['name'] = $abntonames[$agency['abn']][0]; $agency['name'] = $abntonames[$agency['abn']][0];
$agency["lastScraped"] = "1/1/1970"; $agency["lastScraped"] = "1/1/1970";
$agency["scrapeDepth"] = 1; $agency["scrapeDepth"] = 1;
try { try {
$doc = $db->save($agency); $doc = $db->save($agency);
//print_r($doc); //print_r($doc);
echo $agency['abn'] . " imported \n<br>"; echo $agency['abn'] . " imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
?> ?>
   
file:b/admin/refreshDesignDoc.php (new)
  <?php
 
  require_once '../include/common.inc.php';
  $db = $server->get_db('disclosr-agencies');
  createAgencyDesignDoc();
  ?>
 
file:b/admin/resolveConflicts.php (new)
  <?php
 
  include_once('../include/common.inc.php');
  include_header();
  // Include the diff class
  echo '<STYLE TYPE="text/css">
  <!--
  @import url(../lib/php-diff/example/styles.css);
  -->
  </STYLE>
  ';
  require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff.php';
  // Generate a side by side diff
  require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff/Renderer/Html/SideBySide.php';
  $renderer = new Diff_Renderer_Html_SideBySide;
 
 
 
  $db = $server->get_db('disclosr-agencies');
  $docs = Array();
  try {
  $rows = $db->get_view("app", "getConflicts")->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  echo '<h2>' . $row->id . '</h2>';
  echo "Comparing " . $row->value[0] . " and " . $row->value[1];
  $docA = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[0])));
  $docB = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[1])));
  // Options for generating the diff
  $options = array(
  //'ignoreWhitespace' => true,
  //'ignoreCase' => true,
  );
 
  // Initialize the diff class
  $diff = new Diff($docA, $docB, $options);
  echo $diff->Render($renderer);
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
  include_footer();
  ?>
file:a/admin/verify.php -> file:b/admin/verify.php
<?php <?php
   
include_once('include/common.inc.php'); include_once('../include/common.inc.php');
include_header(); include_header();
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
$docs = Array(); $docs = Array();
try { try {
$rows = $db->get_view("app", "byABN")->rows; $rows = $db->get_view("app", "byABN")->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$docs["a" . $row->key] = $row->value; $docs["a" . $row->key] = $row->value;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
//print_r($docs); //print_r($docs);
$row = 1; $row = 1;
if (($handle = fopen("cacfma.csv", "r")) !== FALSE) { if (($handle = fopen("cacfma.csv", "r")) !== FALSE) {
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$row++; $row++;
echo $data[0] . " " . str_replace("?", "", $data[1]) . "<br />\n"; echo $data[0] . " " . str_replace("?", "", $data[1]) . "<br />\n";
$name = $data[0]; $name = $data[0];
$abn = trim(str_replace("?", "", $data[1])); $abn = trim(str_replace("?", "", $data[1]));
$aabn = "a".$abn; $aabn = "a".$abn;
if (isset($docs[$aabn])) { if (isset($docs[$aabn])) {
echo "Existing agency ABN detected<br>"; echo "Existing agency ABN detected<br>";
if (!in_array($name, object_to_array($docs[$aabn]->otherNames)) && $name != $docs[$aabn]->name) { if (!in_array($name, object_to_array($docs[$aabn]->otherNames)) && $name != $docs[$aabn]->name) {
$docs[$aabn]->otherNames[] = $name; $docs[$aabn]->otherNames[] = $name;
try { try {
$docs[$aabn] = $db->save($docs[$aabn]); $docs[$aabn] = $db->save($docs[$aabn]);
//print_r($doc); //print_r($doc);
echo $abn . " additional names imported \n<br>"; echo $abn . " additional names imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
} else { } else {
echo "New agency ABN detected<br>"; echo "New agency ABN detected<br>";
$agency['_id'] = md5($aabn); $agency['_id'] = md5($aabn);
$agency['name'] = $name; $agency['name'] = $name;
$agency["abn"] = $abn; $agency["abn"] = $abn;
try { try {
$doc = $db->save($agency); $doc = $db->save($agency);
print_r($doc); print_r($doc);
echo $abn . " imported \n<br>"; echo $abn . " imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
echo "<hr>"; echo "<hr>";
} }
fclose($handle); fclose($handle);
} }
include_footer(); include_footer();
?> ?>
file:b/alaveteli/exportAgencies.csv.php (new)
  <?php
 
  include_once("../include/common.inc.php");
 
  function shortName($name) {
  $name = trim($name);
  if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
  $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
  return str_replace($badWords,"",$name);
  }
 
  else {
  $out = Array();
  preg_match_all('/[A-Z]/', $name, $out);
  return implode("", $out[0]);
  }
  }
 
  setlocale(LC_CTYPE, 'C');
 
  $headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
 
  $db = $server->get_db('disclosr-agencies');
 
  $tag = Array();
  try {
  $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  die();
  }
 
  $foiEmail = Array();
  try {
  $rows = $db->get_view("app", "foiEmails", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  $foiEmail[$row->key] = $row->value;
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  die();
  }
 
  $fp = fopen('php://output', 'w');
  if ($fp && $db) {
  header('Content-Type: text/csv; charset=utf-8');
  header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
  header('Pragma: no-cache');
  header('Expires: 0');
  fputcsv($fp, $headers);
  try {
  $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
  //print_r($rows);
  foreach ($agencies as $agency) {
  // print_r($agency);
 
  if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
  $row = Array();
  $row["#id"] = $agency->id;
  $row["name"] = trim($agency->value->name);
  if (isset($agency->value->foiEmail)) {
  $row["request_email"] = $agency->value->foiEmail;
  } else {
  if ($agency->value->orgType == "FMA-DepartmentOfState") {
  $row["request_email"] = "foi@" . GetDomain($agency->value->website);
  } else {
  $row["request_email"] = $foiEmail[$agency->value->parentOrg];
  }
  }
  if (isset($agency->value->shortName)) {
  $row["short_name"] = $agency->value->shortName;
  } else {
  $row["short_name"] = shortName($agency->value->name);
  }
  $row["notes"] = "";
  $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
  $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
  if ($agency->value->orgType == "FMA-DepartmentOfState") {
  $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
  } else {
  $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
  }
 
  fputcsv($fp, array_values($row));
 
  if (isset($agency->value->foiBodies)) {
  foreach ($agency->value->foiBodies as $foiBody) {
  $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
  $row["short_name"] = shortName($foiBody);
  fputcsv($fp, array_values($row));
  }
  }
  }
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  die;
  }
  ?>
 
file:b/alaveteli/exportCategories.rb.php (new)
  <?php
  include_once("../include/common.inc.php");
  setlocale(LC_CTYPE, 'C');
  header('Content-Type: text/csv');
  header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
  header('Pragma: no-cache');
  header('Expires: 0');
  echo 'PublicBodyCategories.add(:en, ['.PHP_EOL;
  echo ' "Portfolios",'.PHP_EOL;
  $db = $server->get_db('disclosr-agencies');
 
  try {
  $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  echo ' [ "'.phrase_to_tag(dept_to_portfolio($row->key)).'","'. dept_to_portfolio($row->key).'","part of the '.dept_to_portfolio($row->key).' portfolio" ],'.PHP_EOL;
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
  echo '])';
  ?>
 
file:a/getAgency.php -> file:b/getAgency.php
<?php <?php
   
include_once('include/common.inc.php'); include_once('include/common.inc.php');
include_header(); include_header();
   
function displayValue($key, $value, $mode) { function displayValue($key, $value, $mode) {
global $db; global $db, $schemas;
if ($mode == "view") { if ($mode == "view") {
   
  echo "<tr>";
   
  echo "<td>" . $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small></td><td>";
if (is_array($value)) { if (is_array($value)) {
echo "<tr><td>$key</td><td><ol>"; echo "<ol>";
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
echo "<li>$subvalue</li>"; if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
  echo '<li itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
  } else {
  echo "<li>";
  }
  echo "$subvalue</li>";
} }
echo "</ol></td></tr>"; echo "</ol></td></tr>";
} else { } else {
echo "<tr><td>$key</td><td>$value</td></tr>"; if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
  echo '<span itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
  } else {
  echo "<span>";
  }
  if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
  echo "<a href='$value'>view</a></span>";
  } else {
  echo "$value</span>";
  }
} }
  echo "</td></tr>";
} }
if ($mode == "edit") { if ($mode == "edit") {
if (is_array($value)) { if (is_array($value)) {
echo '<div class="row"> echo '<div class="row">
<div class="seven columns"> <div class="seven columns">
<fieldset> <fieldset>
<h5>' . $key . '</h5>'; <h5>' . $key . '</h5>';
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>"; echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>";
}