projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Handling of minister/secretary names in FOI export
Handling of minister/secretary names in FOI export


Former-commit-id: 4d6a601bd2ae9012300836e1ddf12bc147981e10

file:a/alaveteli/exportAgencies.csv.php -> file:b/alaveteli/exportAgencies.csv.php
<?php <?php
   
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
   
  function shortName($name) {
  $name = trim($name);
  if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
  $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
  return str_replace($badWords,"",$name);
  }
   
  else {
  $out = Array();
  preg_match_all('/[A-Z]/', $name, $out);
  return implode("", $out[0]);
  }
  }
   
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
   
$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string"); $headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
   
$tag = Array(); $tag = Array();
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key)); $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
  die();
} }
   
$foiEmail = Array(); $foiEmail = Array();
try { try {
$rows = $db->get_view("app", "foiEmails", null, true)->rows; $rows = $db->get_view("app", "foiEmails", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
$foiEmail[$row->key] = $row->value; $foiEmail[$row->key] = $row->value;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
  die();
} }
   
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $db) { if ($fp && $db) {
header('Content-Type: text/csv'); header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
fputcsv($fp, $headers); fputcsv($fp, $headers);
try { try {
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($agencies as $agency) { foreach ($agencies as $agency) {
// print_r($agency); // print_r($agency);
   
if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) { if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
$row = Array(); $row = Array();
$row["#id"] = $agency->id; $row["#id"] = $agency->id;
$row["name"] = trim($agency->value->name); $row["name"] = trim($agency->value->name);
if (isset($agency->value->foiEmail)) { if (isset($agency->value->foiEmail)) {
$row["request_email"] = $agency->value->foiEmail; $row["request_email"] = $agency->value->foiEmail;
} else { } else {
if ($agency->value->orgType == "FMA-DepartmentOfState") { if ($agency->value->orgType == "FMA-DepartmentOfState") {
$row["request_email"] = "foi@" . GetDomain($agency->value->website); $row["request_email"] = "foi@" . GetDomain($agency->value->website);
} else { } else {
$row["request_email"] = $foiEmail[$agency->value->parentOrg]; $row["request_email"] = $foiEmail[$agency->value->parentOrg];
} }
} }
if (isset($agency->value->shortName)) { if (isset($agency->value->shortName)) {
$row["short_name"] = $agency->value->shortName; $row["short_name"] = $agency->value->shortName;
} else { } else {
$out = Array(); $row["short_name"] = shortName($agency->value->name);
preg_match_all('/[A-Z]/', trim($agency->value->name), $out);  
$row["short_name"] = implode("", $out[0]);  
} }
$row["notes"] = ""; $row["notes"] = "";
$row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : ""); $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
$row["home_page"] = (isset($agency->value->website) ? $agency->value->website : ""); $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
if ($agency->value->orgType == "FMA-DepartmentOfState") { if ($agency->value->orgType == "FMA-DepartmentOfState") {
$row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType; $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
} else { } else {
$row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType; $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
} }
   
fputcsv($fp, array_values($row)); fputcsv($fp, array_values($row));
   
if (isset($agency->value->foiBodies)) { if (isset($agency->value->foiBodies)) {
foreach ($agency->value->foiBodies as $foiBody) { foreach ($agency->value->foiBodies as $foiBody) {
$row['name'] = $foiBody; $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
$row['short_name'] = ""; $row["short_name"] = shortName($foiBody);
fputcsv($fp, array_values($row)); fputcsv($fp, array_values($row));
} }
} }
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
die; die;
} }
?> ?>
   
file:a/include/common.inc.php -> file:b/include/common.inc.php
<?php <?php
   
date_default_timezone_set("Australia/Sydney"); date_default_timezone_set("Australia/Sydney");
   
$basePath = ""; $basePath = "";
if (strstr($_SERVER['PHP_SELF'], "alaveteli/") if (strstr($_SERVER['PHP_SELF'], "alaveteli/")
|| strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "admin/")
|| strstr($_SERVER['PHP_SELF'], "lib/") || strstr($_SERVER['PHP_SELF'], "lib/")
|| strstr($_SERVER['PHP_SELF'], "include/")) || strstr($_SERVER['PHP_SELF'], "include/"))
$basePath = "../"; $basePath = "../";
   
include_once ('couchdb.inc.php'); include_once ('couchdb.inc.php');
include_once ('template.inc.php'); include_once ('template.inc.php');
   
# Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735 # Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735
   
function object_to_array(stdClass $Class) { function object_to_array(stdClass $Class) {
# Typecast to (array) automatically converts stdClass -> array. # Typecast to (array) automatically converts stdClass -> array.
$Class = (array) $Class; $Class = (array) $Class;
   
# Iterate through the former properties looking for any stdClass properties. # Iterate through the former properties looking for any stdClass properties.
# Recursively apply (array). # Recursively apply (array).
foreach ($Class as $key => $value) { foreach ($Class as $key => $value) {
if (is_object($value) && get_class($value) === 'stdClass') { if (is_object($value) && get_class($value) === 'stdClass') {
$Class[$key] = object_to_array($value); $Class[$key] = object_to_array($value);
} }
} }
return $Class; return $Class;
} }
   
# Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735 # Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735
   
function array_to_object(array $array) { function array_to_object(array $array) {
# Iterate through our array looking for array values. # Iterate through our array looking for array values.
# If found recurvisely call itself. # If found recurvisely call itself.
foreach ($array as $key => $value) { foreach ($array as $key => $value) {
if (is_array($value)) { if (is_array($value)) {
$array[$key] = array_to_object($value); $array[$key] = array_to_object($value);
} }
} }
   
# Typecast to (object) will automatically convert array -> stdClass # Typecast to (object) will automatically convert array -> stdClass
return (object) $array; return (object) $array;
} }
   
function dept_to_portfolio($deptName) { function dept_to_portfolio($deptName) {
return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName))); return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName)));
} }
function phrase_to_tag ($phrase) { function phrase_to_tag ($phrase) {
return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase)))); return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase))));
} }
function GetDomain($url) function GetDomain($url)
{ {
$nowww = ereg_replace('www\.','',$url); $nowww = ereg_replace('www\.','',$url);
$domain = parse_url($nowww); $domain = parse_url($nowww);
if(!empty($domain["host"])) if(!empty($domain["host"]))
{ {
return $domain["host"]; return $domain["host"];
} else } else
{ {
return $domain["path"]; return $domain["path"];
} }
} }
   
?>  
   
   
   
file:a/include/couchdb.inc.php -> file:b/include/couchdb.inc.php
<?php <?php
   
include $basePath."schemas/schemas.inc.php"; include $basePath."schemas/schemas.inc.php";
   
require ($basePath.'couchdb/settee/src/settee.php'); require ($basePath.'couchdb/settee/src/settee.php');
   
function createAgencyDesignDoc() { function createAgencyDesignDoc() {
global $db; global $db;
$obj = new stdClass(); $obj = new stdClass();
$obj->_id = "_design/" . urlencode("app"); $obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript"; $obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->byCanonicalName->map = "function(doc) { $obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') { if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc); emit(doc.name, doc);
} }
};"; };";
$obj->views->byDeptStateName->map = "function(doc) { $obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') { if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id); emit(doc.name, doc._id);
} }
};"; };";
$obj->views->parentOrgs->map = "function(doc) { $obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) { if (doc.parentOrg) {
emit(doc._id, doc.parentOrg); emit(doc._id, doc.parentOrg);
} }
};"; };";
$obj->views->byName->map = "function(doc) { $obj->views->byName->map = "function(doc) {
emit(doc.name, doc._id); emit(doc.name, doc._id);
for (name in doc.otherNames) { for (name in doc.otherNames) {
if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) {
emit(doc.otherNames[name], doc._id); emit(doc.otherNames[name], doc._id);
} }
} }
};"; };";
$obj->views->foiEmails->map = "function(doc) { $obj->views->foiEmails->map = "function(doc) {
emit(doc._id, doc.foiEmail); emit(doc._id, doc.foiEmail);
};"; };";
$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }"; $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };'; $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };'; $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
$obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };"; $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
$obj->views->getConflicts->map = "function(doc) { $obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) { if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts)); emit(null, [doc._rev].concat(doc._conflicts));
} }
}"; }";
// http://stackoverflow.com/questions/646628/javascript-startswith // http://stackoverflow.com/questions/646628/javascript-startswith
$obj->views->score->map = 'if(!String.prototype.startsWith){ $obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) { String.prototype.startsWith = function (str) {
return !this.indexOf(str); return !this.indexOf(str);
} }
} }
   
function(doc) { function(doc) {
count = 0; count = 0;
if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
for(var propName in doc) { for(var propName in doc) {
if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) { if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) {
count++ count++
} }
} }
emit(count+doc._id, {id:doc._id, name: doc.name, score:count}); emit(count+doc._id, {id:doc._id, name: doc.name, score:count});
} }
}'; }';
   
// allow safe updates (even if slightly slower due to extra: rev-detection check). // allow safe updates (even if slightly slower due to extra: rev-detection check).
return $db->save($obj, true); return $db->save($obj, true);
} }
   
   
if( php_uname('n') == "vanille") { if( php_uname('n') == "vanille") {
   
$server = new SetteeServer('http://192.168.178.21:5984'); $server = new SetteeServer('http://192.168.178.21:5984');
} else } else
if( php_uname('n') == "KYUUBEY") { if( php_uname('n') == "KYUUBEY") {
   
$server = new SetteeServer('http://192.168.1.148:5984'); $server = new SetteeServer('http://192.168.1.148:5984');
} else { } else {
$server = new SetteeServer('http://127.0.0.1:5984'); $server = new SetteeServer('http://127.0.0.1:5984');
} }
function setteErrorHandler($e) { function setteErrorHandler($e) {
echo $e->getMessage() . "<br>" . PHP_EOL; echo $e->getMessage() . "<br>" . PHP_EOL;
} }
?>  
   
file:a/include/template.inc.php -> file:b/include/template.inc.php
<?php <?php
   
function include_header() { function include_header() {
global $basePath; global $basePath;
?> ?>
<!DOCTYPE html> <!DOCTYPE html>
   
<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ --> <!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]--> <!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]--> <!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]--> <!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
<!--[if gt IE 8]><!--> <html lang="en"> <!--<![endif]--> <!--[if gt IE 8]><!--> <html lang="en"> <!--<![endif]-->
<head> <head>
<meta charset="utf-8" /> <meta charset="utf-8" />
   
<!-- Set the viewport width to device width for mobile --> <!-- Set the viewport width to device width for mobile -->
<meta name="viewport" content="width=device-width" /> <meta name="viewport" content="width=device-width" />
   
<title>Disclosr</title> <title>Disclosr</title>
   
<!-- Included CSS Files --> <!-- Included CSS Files -->
<link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/foundation.css"> <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/foundation.css">
<link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/app.css"> <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/app.css">
   
<!--[if lt IE 9]> <!--[if lt IE 9]>
<link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/ie.css"> <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/ie.css">
<![endif]--> <![endif]-->
   
   
<!-- IE Fix for HTML5 Tags --> <!-- IE Fix for HTML5 Tags -->
<!--[if lt IE 9]> <!--[if lt IE 9]>
<script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script> <script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]--> <![endif]-->
   
</head> </head>
<body> <body>
   
<!-- navBar --> <!-- navBar -->
<div id="navbar" class="container"> <div id="navbar" class="container">
<div class="row"> <div class="row">
<div class="four columns"> <div class="four columns">
<h1><a href="/">Disclosr</a></h1> <h1><a href="/">Disclosr</a></h1>
</div> </div>
<div class="eight columns hide-on-phones"> <div class="eight columns hide-on-phones">
<strong class="right"> <strong class="right">
<a href="getAgency.php">Agencies</a> <a href="getAgency.php">Agencies</a>
<a href="about.php">About/FAQ</a> <a href="about.php">About/FAQ</a>
</strong> </strong>
</div> </div>
</div> </div>
</div> </div>
<!-- /navBar -->