projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Sub-agency FOI export for alaveteli
Sub-agency FOI export for alaveteli


Former-commit-id: 546acd4a1b6f65733606771bd7152278e85f1594

file:a/.gitmodules -> file:b/.gitmodules
[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"] [submodule "couchdb/settee"]
path = couchdb/settee path = couchdb/settee
url = https://github.com/inadarei/settee.git url = https://github.com/inadarei/settee.git
[submodule "lib/springy"] [submodule "lib/springy"]
path = lib/springy path = lib/springy
url = https://github.com/dhotson/springy.git url = https://github.com/dhotson/springy.git
  [submodule "lib/php-diff"]
  path = lib/php-diff
  url = https://github.com/chrisboulton/php-diff.git
   
file:a/admin/import.php -> file:b/admin/import.php
<?php <?php
   
require_once 'include/common.inc.php'; require_once '../include/common.inc.php';
try { try {
$server->create_db('disclosr-agencies'); $server->create_db('disclosr-agencies');
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
createAgencyDesignDoc(); createAgencyDesignDoc();
$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost"); $conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");
$namesQ = 'select agency.abn, string_agg("agencyName",\'|\') as names from agency inner join agency_nametoabn on agency.abn::text = agency_nametoabn.abn group by agency.abn;'; $namesQ = 'select agency.abn, string_agg("agencyName",\'|\') as names from agency inner join agency_nametoabn on agency.abn::text = agency_nametoabn.abn group by agency.abn;';
$abntonames = Array(); $abntonames = Array();
foreach ($conn->query($namesQ) as $row) { foreach ($conn->query($namesQ) as $row) {
$abntonames[$row['abn']] = explode("|", $row['names']); $abntonames[$row['abn']] = explode("|", $row['names']);
} }
$result = $conn->query("select * from agency"); $result = $conn->query("select * from agency");
while ($agency = $result->fetch(PDO::FETCH_ASSOC)) { while ($agency = $result->fetch(PDO::FETCH_ASSOC)) {
$agency['_id'] = md5($agency['abn']); $agency['_id'] = md5($agency['abn']);
$agency['otherNames'] = $abntonames[$agency['abn']]; $agency['otherNames'] = $abntonames[$agency['abn']];
if (sizeof($abntonames[$agency['abn']]) == 1) if (sizeof($abntonames[$agency['abn']]) == 1)
$agency['name'] = $abntonames[$agency['abn']][0]; $agency['name'] = $abntonames[$agency['abn']][0];
$agency["lastScraped"] = "1/1/1970"; $agency["lastScraped"] = "1/1/1970";
$agency["scrapeDepth"] = 1; $agency["scrapeDepth"] = 1;
try { try {
$doc = $db->save($agency); $doc = $db->save($agency);
//print_r($doc); //print_r($doc);
echo $agency['abn'] . " imported \n<br>"; echo $agency['abn'] . " imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
?> ?>
   
file:b/admin/refreshDesignDoc.php (new)
  <?php
 
  require_once '../include/common.inc.php';
  $db = $server->get_db('disclosr-agencies');
  createAgencyDesignDoc();
  ?>
 
file:b/admin/resolveConflicts.php (new)
  <?php
 
  include_once('../include/common.inc.php');
  include_header();
  // Include the diff class
  echo '<STYLE TYPE="text/css">
  <!--
  @import url(../lib/php-diff/example/styles.css);
  -->
  </STYLE>
  ';
  require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff.php';
  // Generate a side by side diff
  require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff/Renderer/Html/SideBySide.php';
  $renderer = new Diff_Renderer_Html_SideBySide;
 
 
 
  $db = $server->get_db('disclosr-agencies');
  $docs = Array();
  try {
  $rows = $db->get_view("app", "getConflicts")->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  echo '<h2>' . $row->id . '</h2>';
  echo "Comparing " . $row->value[0] . " and " . $row->value[1];
  $docA = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[0])));
  $docB = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[1])));
  // Options for generating the diff
  $options = array(
  //'ignoreWhitespace' => true,
  //'ignoreCase' => true,
  );
 
  // Initialize the diff class
  $diff = new Diff($docA, $docB, $options);
  echo $diff->Render($renderer);
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
  include_footer();
  ?>
file:a/admin/verify.php -> file:b/admin/verify.php
<?php <?php
   
include_once('include/common.inc.php'); include_once('../include/common.inc.php');
include_header(); include_header();
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
$docs = Array(); $docs = Array();
try { try {
$rows = $db->get_view("app", "byABN")->rows; $rows = $db->get_view("app", "byABN")->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$docs["a" . $row->key] = $row->value; $docs["a" . $row->key] = $row->value;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
//print_r($docs); //print_r($docs);
$row = 1; $row = 1;
if (($handle = fopen("cacfma.csv", "r")) !== FALSE) { if (($handle = fopen("cacfma.csv", "r")) !== FALSE) {
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$row++; $row++;
echo $data[0] . " " . str_replace("?", "", $data[1]) . "<br />\n"; echo $data[0] . " " . str_replace("?", "", $data[1]) . "<br />\n";
$name = $data[0]; $name = $data[0];
$abn = trim(str_replace("?", "", $data[1])); $abn = trim(str_replace("?", "", $data[1]));
$aabn = "a".$abn; $aabn = "a".$abn;
if (isset($docs[$aabn])) { if (isset($docs[$aabn])) {
echo "Existing agency ABN detected<br>"; echo "Existing agency ABN detected<br>";
if (!in_array($name, object_to_array($docs[$aabn]->otherNames)) && $name != $docs[$aabn]->name) { if (!in_array($name, object_to_array($docs[$aabn]->otherNames)) && $name != $docs[$aabn]->name) {
$docs[$aabn]->otherNames[] = $name; $docs[$aabn]->otherNames[] = $name;
try { try {
$docs[$aabn] = $db->save($docs[$aabn]); $docs[$aabn] = $db->save($docs[$aabn]);
//print_r($doc); //print_r($doc);
echo $abn . " additional names imported \n<br>"; echo $abn . " additional names imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
} else { } else {
echo "New agency ABN detected<br>"; echo "New agency ABN detected<br>";
$agency['_id'] = md5($aabn); $agency['_id'] = md5($aabn);
$agency['name'] = $name; $agency['name'] = $name;
$agency["abn"] = $abn; $agency["abn"] = $abn;
try { try {
$doc = $db->save($agency); $doc = $db->save($agency);
print_r($doc); print_r($doc);
echo $abn . " imported \n<br>"; echo $abn . " imported \n<br>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
echo "<hr>"; echo "<hr>";
} }
fclose($handle); fclose($handle);
} }
include_footer(); include_footer();
?> ?>
file:a/alaveteli/exportAgencies.csv.php -> file:b/alaveteli/exportAgencies.csv.php
<?php <?php
   
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
   
$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string"); $headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
   
$tag = Array(); $tag = Array();
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$tag[$row->value] = phrase_to_tag(dept_to_portfolio($row->key)); $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
   
  $foiEmail = Array();
  try {
  $rows = $db->get_view("app", "foiEmails", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  $foiEmail[$row->key] = $row->value;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $db) { if ($fp && $db) {
header('Content-Type: text/csv'); header('Content-Type: text/csv');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
fputcsv($fp, $headers); fputcsv($fp, $headers);
try { try {
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($agencies as $agency) { foreach ($agencies as $agency) {
// print_r($agency); // print_r($agency);
$row = Array();  
$row["#id"] = $agency->value->_id; if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
$row["name"] = $agency->value->name; $row = Array();
if (isset($agency->value->foi_email)) { $row["#id"] = $agency->id;
$row["request_email"] = $agency->value->foi_email; $row["name"] = trim($agency->value->name);
} else { if (isset($agency->value->foiEmail)) {
$row["request_email"] = "foi@".GetDomain($agency->value->website); $row["request_email"] = $agency->value->foiEmail;
// } else {
} if ($agency->value->orgType == "FMA-DepartmentOfState") {
if (isset($agency->value->short_name)) { $row["request_email"] = "foi@" . GetDomain($agency->value->website);
$row["short_name"] = $agency->value->short_name; } else {
} else { $row["request_email"] = $foiEmail[$agency->value->parentOrg];
$out = Array(); }
preg_match_all('/[A-Z]/', $agency->value->name, $out); }
$row["short_name"] = implode("",$out[0]); if (isset($agency->value->shortName)) {
} $row["short_name"] = $agency->value->shortName;
$row["notes"] = ""; } else {
$row["publication_scheme"] = $agency->value->infoPublicationSchemeURL; $out = Array();
$row["home_page"] = $agency->value->website; preg_match_all('/[A-Z]/', trim($agency->value->name), $out);
if ($agency->value->orgType == "FMA-DepartmentOfState") { $row["short_name"] = implode("", $out[0]);
$row["tag_string"] = $tag[$agency->value->_id]; }
} else { $row["notes"] = "";
$row["tag_string"] = $tag[$agency->value->parentOrg];; $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
} $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
  if ($agency->value->orgType == "FMA-DepartmentOfState") {
fputcsv($fp, array_values($row)); $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
  } else {
if (isset($agency->value->foiBodies)) { $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
foreach ($agency->value->foiBodies as $foiBody) { }
$row['name'] = $foiBody;  
$row['short_name'] = ""; fputcsv($fp, array_values($row));
fputcsv($fp, array_values($row));  
  if (isset($agency->value->foiBodies)) {
  foreach ($agency->value->foiBodies as $foiBody) {
  $row['name'] = $foiBody;
  $row['short_name'] = "";
  fputcsv($fp, array_values($row));
  }
} }
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
die; die;
} }
?> ?>
   
file:a/alaveteli/exportCategories.rb.php -> file:b/alaveteli/exportCategories.rb.php
<?php <?php
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
header('Content-Type: text/csv'); header('Content-Type: text/csv');
header('Content-Disposition: attachment; filename="public_body_categories_en.rb"'); header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
echo 'PublicBodyCategories.add(:en, ['.PHP_EOL; echo 'PublicBodyCategories.add(:en, ['.PHP_EOL;
echo ' "Portfolios",'.PHP_EOL; echo ' "Portfolios",'.PHP_EOL;
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
   
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo ' [ "'.phrase_to_tag(dept_to_portfolio($row->key)).'","'. dept_to_portfolio($row->key).'","'.dept_to_portfolio($row->key).'" ],'.PHP_EOL; echo ' [ "'.phrase_to_tag(dept_to_portfolio($row->key)).'","'. dept_to_portfolio($row->key).'","part of the '.dept_to_portfolio($row->key).' portfolio" ],'.PHP_EOL;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
echo '])'; echo '])';
?> ?>
   
file:a/getAgency.php -> file:b/getAgency.php
<?php <?php
   
include_once('include/common.inc.php'); include_once('include/common.inc.php');
include_header(); include_header();
   
function displayValue($key, $value, $mode) { function displayValue($key, $value, $mode) {
global $db; global $db;
if ($mode == "view") { if ($mode == "view") {
if (is_array($value)) { if (is_array($value)) {
echo "<tr><td>$key</td><td><ol>"; echo "<tr><td>$key</td><td><ol>";
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
echo "<li>$subvalue</li>"; echo "<li>$subvalue</li>";
} }
echo "</ol></td></tr>"; echo "</ol></td></tr>";
} else { } else {
echo "<tr><td>$key</td><td>$value</td></tr>"; echo "<tr><td>$key</td><td>$value</td></tr>";
} }
} }
if ($mode == "edit") { if ($mode == "edit") {
if (is_array($value)) { if (is_array($value)) {
echo '<div class="row"> echo '<div class="row">
<div class="seven columns"> <div class="seven columns">
<fieldset> <fieldset>
<h5>' . $key . '</h5>'; <h5>' . $key . '</h5>';
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>"; echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>";
} }
echo "</fieldset> echo "</fieldset>
</div> </div>
</div>"; </div>";
} else { } else {
if (strpos($key, "_") === 0) { if (strpos($key, "_") === 0) {
echo"<input type='hidden' id='$key' name='$key' value='$value'/>"; echo"<input type=