projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Handling of minister/secretary names in FOI export
Handling of minister/secretary names in FOI export


Former-commit-id: 4d6a601bd2ae9012300836e1ddf12bc147981e10

file:a/alaveteli/exportAgencies.csv.php -> file:b/alaveteli/exportAgencies.csv.php
<?php <?php
   
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
   
  function shortName($name) {
  $name = trim($name);
  if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
  $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
  return str_replace($badWords,"",$name);
  }
   
  else {
  $out = Array();
  preg_match_all('/[A-Z]/', $name, $out);
  return implode("", $out[0]);
  }
  }
   
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
   
$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string"); $headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
   
$tag = Array(); $tag = Array();
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$tag[$row->value] = phrase_to_tag(dept_to_portfolio($row->key)); $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
  die();
  }
   
  $foiEmail = Array();
  try {
  $rows = $db->get_view("app", "foiEmails", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  $foiEmail[$row->key] = $row->value;
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  die();
} }
   
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $db) { if ($fp && $db) {
header('Content-Type: text/csv'); header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
fputcsv($fp, $headers); fputcsv($fp, $headers);
try { try {
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($agencies as $agency) { foreach ($agencies as $agency) {
// print_r($agency); // print_r($agency);
$row = Array();  
$row["#id"] = $agency->value->_id; if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
$row["name"] = $agency->value->name; $row = Array();
if (isset($agency->value->foi_email)) { $row["#id"] = $agency->id;
$row["request_email"] = $agency->value->foiEmail; $row["name"] = trim($agency->value->name);
} else { if (isset($agency->value->foiEmail)) {
$row["request_email"] = "foi@".GetDomain($agency->value->website); $row["request_email"] = $agency->value->foiEmail;
// } else {
} if ($agency->value->orgType == "FMA-DepartmentOfState") {
if (isset($agency->value->shortName)) { $row["request_email"] = "foi@" . GetDomain($agency->value->website);
$row["short_name"] = $agency->value->shortName; } else {
} else { $row["request_email"] = $foiEmail[$agency->value->parentOrg];
$out = Array(); }
preg_match_all('/[A-Z]/', $agency->value->name, $out); }
$row["short_name"] = implode("",$out[0]); if (isset($agency->value->shortName)) {
} $row["short_name"] = $agency->value->shortName;
$row["notes"] = ""; } else {
$row["publication_scheme"] = $agency->value->infoPublicationSchemeURL; $row["short_name"] = shortName($agency->value->name);
$row["home_page"] = $agency->value->website; }
if ($agency->value->orgType == "FMA-DepartmentOfState") { $row["notes"] = "";
$row["tag_string"] = $tag[$agency->value->_id]; $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
} else { $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
$row["tag_string"] = $tag[$agency->value->parentOrg];; if ($agency->value->orgType == "FMA-DepartmentOfState") {
} $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
  } else {
fputcsv($fp, array_values($row)); $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
  }
if (isset($agency->value->foiBodies)) {  
foreach ($agency->value->foiBodies as $foiBody) { fputcsv($fp, array_values($row));
$row['name'] = $foiBody;  
$row['short_name'] = ""; if (isset($agency->value->foiBodies)) {
fputcsv($fp, array_values($row)); foreach ($agency->value->foiBodies as $foiBody) {
  $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
  $row["short_name"] = shortName($foiBody);
  fputcsv($fp, array_values($row));
  }
} }
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
die; die;
} }
?> ?>
   
file:a/graph.php -> file:b/graph.php
<?php <?php
include_once('include/common.inc.php'); include_once('include/common.inc.php');
//include_header(); //include_header();
?> $format = "html";
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script> if (isset($_REQUEST['format'])) {
<script src="lib/springy/springy.js"></script> $format = $_REQUEST['format'];
<script src="lib/springy/springyui.js"></script> }
<script>  
var graph = new Graph(); function add_node($id, $label) {
var nodes = []; global $format;
<?php if ($format == "html") {
  echo "nodes[\"$id\"] = graph.newNode({label: \"$label\"});" . PHP_EOL;
  }
  if ($format == "dot" && $label != "") {
  echo "$id [label=\"$label\"];". PHP_EOL;
  }
  }
   
  function add_edge($from, $to, $color) {
  global $format;
  if ($format == "html") {
  echo "graph.newEdge(nodes[\"$from\"], nodes['$to'], {color: '$color'});" . PHP_EOL;
  }
  if ($format == "dot") {
  echo "$from -> $to ".($color != ""? "[color=$color]":"").";". PHP_EOL;
  }
  }
   
  if ($format == "html") {
  ?>
  <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script>
  <script src="lib/springy/springy.js"></script>
  <script src="lib/springy/springyui.js"></script>
  <script>
  var graph = new Graph();
  var nodes = [];
  <?php
  }
  if ($format == "dot") {
  echo 'digraph g {'. PHP_EOL;
  }
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
echo "nodes[\"fedg\"] = graph.newNode({label: \"Federal Government - Commonwealth of Australia\"});" . PHP_EOL; add_node("fedg","Federal Government - Commonwealth of Australia");
try { try {
$rows = $db->get_view("app", "byCanonicalName", null, true)->rows; $rows = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "nodes[\"{$row->id}\"] = graph.newNode({label: \"{$row->key}\"});" . PHP_EOL; add_node($row->id, $row->key);
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "graph.newEdge(nodes[\"fedg\"], nodes['{$row->value}'], {color: '#00A0B0'});" . PHP_EOL; add_edge("fedg", $row->value, 'yellow');
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
try { try {
$rows = $db->get_view("app", "parentOrgs", null, true)->rows; $rows = $db->get_view("app", "parentOrgs", null, true)->rows;
// print_r($rows); // print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "graph.newEdge(nodes[\"{$row->key}\"], nodes['{$row->value}'], {color: '#FFA0B0'});" . PHP_EOL; add_edge($row->key, $row->value, 'blue');
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
?> if ($format == "html") {
window.onload = function() { ?>
$(document).ready(function() { window.onload = function() {
var springy = $('#springydemo').springy({ $(document).ready(function() {
graph: graph var springy = $('#springydemo').springy({
  graph: graph
  });
}); });
}); };
}; </script>
</script>  
   
<canvas id="springydemo" width="1260" height="680" /> <canvas id="springydemo" width="1260" height="680" />
<?php <?php
  }
  if ($format == "dot") {
  echo "}";
  }
//include_footer(); //include_footer();
?> ?>
   
   
file:a/include/common.inc.php -> file:b/include/common.inc.php
<?php <?php
   
date_default_timezone_set("Australia/Sydney"); date_default_timezone_set("Australia/Sydney");
   
$basePath = ""; $basePath = "";
if (strstr($_SERVER['PHP_SELF'], "alaveteli/") if (strstr($_SERVER['PHP_SELF'], "alaveteli/")
|| strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "admin/")
|| strstr($_SERVER['PHP_SELF'], "lib/") || strstr($_SERVER['PHP_SELF'], "lib/")
|| strstr($_SERVER['PHP_SELF'], "include/")) || strstr($_SERVER['PHP_SELF'], "include/"))
$basePath = "../"; $basePath = "../";
   
include_once ('couchdb.inc.php'); include_once ('couchdb.inc.php');
include_once ('template.inc.php'); include_once ('template.inc.php');
   
# Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735 # Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735
   
function object_to_array(stdClass $Class) { function object_to_array(stdClass $Class) {
# Typecast to (array) automatically converts stdClass -> array. # Typecast to (array) automatically converts stdClass -> array.
$Class = (array) $Class; $Class = (array) $Class;
   
# Iterate through the former properties looking for any stdClass properties. # Iterate through the former properties looking for any stdClass properties.
# Recursively apply (array). # Recursively apply (array).
foreach ($Class as $key => $value) { foreach ($Class as $key => $value) {
if (is_object($value) && get_class($value) === 'stdClass') { if (is_object($value) && get_class($value) === 'stdClass') {
$Class[$key] = object_to_array($value); $Class[$key] = object_to_array($value);
} }
} }
return $Class; return $Class;
} }
   
# Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735 # Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735
   
function array_to_object(array $array) { function array_to_object(array $array) {
# Iterate through our array looking for array values. # Iterate through our array looking for array values.
# If found recurvisely call itself. # If found recurvisely call itself.
foreach ($array as $key => $value) { foreach ($array as $key => $value) {
if (is_array($value)) { if (is_array($value)) {
$array[$key] = array_to_object($value); $array[$key] = array_to_object($value);
} }
} }
   
# Typecast to (object) will automatically convert array -> stdClass # Typecast to (object) will automatically convert array -> stdClass
return (object) $array; return (object) $array;
} }
   
function dept_to_portfolio($deptName) { function dept_to_portfolio($deptName) {
return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName))); return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName)));
} }
function phrase_to_tag ($phrase) { function phrase_to_tag ($phrase) {
return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase)))); return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase))));
} }
function GetDomain($url) function GetDomain($url)
{ {
$nowww = ereg_replace('www\.','',$url); $nowww = ereg_replace('www\.','',$url);
$domain = parse_url($nowww); $domain = parse_url($nowww);
if(!empty($domain["host"])) if(!empty($domain["host"]))
{ {
return $domain["host"]; return $domain["host"];
} else } else
{ {
return $domain["path"]; return $domain["path"];
} }
} }
   
?>  
   
   
   
file:a/include/couchdb.inc.php -> file:b/include/couchdb.inc.php
<?php <?php
   
include $basePath."schemas/schemas.inc.php"; include $basePath."schemas/schemas.inc.php";
   
require ($basePath.'couchdb/settee/src/settee.php'); require ($basePath.'couchdb/settee/src/settee.php');
   
function createAgencyDesignDoc() { function createAgencyDesignDoc() {
global $db; global $db;
$obj = new stdClass(); $obj = new stdClass();
$obj->_id = "_design/" . urlencode("app"); $obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript"; $obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->byCanonicalName->map = "function(doc) { $obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') { if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id); emit(doc.name, doc);
} }
};"; };";
$obj->views->byDeptStateName->map = "function(doc) { $obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') { if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id); emit(doc.name, doc._id);
} }
};"; };";
$obj->views->parentOrgs->map = "function(doc) { $obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) { if (doc.parentOrg) {
emit(doc._id, doc.parentOrg); emit(doc._id, doc.parentOrg);
} }
};"; };";
$obj->views->byName->map = "function(doc) { $obj->views->byName->map = "function(doc) {
emit(doc.name, doc._id); emit(doc.name, doc._id);
for (name in doc.otherNames) { for (name in doc.otherNames) {
if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) {
emit(doc.otherNames[name], doc._id); emit(doc.otherNames[name], doc._id);
} }
} }
};"; };";
   
  $obj->views->foiEmails->map = "function(doc) {
  emit(doc._id, doc.foiEmail);
  };";
   
$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }"; $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };'; $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };'; $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
$obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };"; $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
$obj->views->getConflicts->map = "function(doc) { $obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) { if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts)); emit(null, [doc._rev].concat(doc._conflicts));
} }
}"; }";
// http://stackoverflow.com/questions/646628/javascript-startswith // http://stackoverflow.com/questions/646628/javascript-startswith
$obj->views->score->map = 'if(!String.prototype.startsWith){