projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Data export for alaveteli
Data export for alaveteli


Former-commit-id: 94980807a4ab6705611f602879d6b7dd0e7e62f1

file:b/alaveteli/exportAgencies.csv.php (new)
  <?php
 
  include_once("../include/common.inc.php");
  setlocale(LC_CTYPE, 'C');
 
  $headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
 
  $db = $server->get_db('disclosr-agencies');
 
  $tag = Array();
  try {
  $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  $tag[$row->value] = phrase_to_tag(dept_to_portfolio($row->key));
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  $fp = fopen('php://output', 'w');
  if ($fp && $db) {
  header('Content-Type: text/csv');
  header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
  header('Pragma: no-cache');
  header('Expires: 0');
  fputcsv($fp, $headers);
  try {
  $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
  //print_r($rows);
  foreach ($agencies as $agency) {
  // print_r($agency);
  $row = Array();
  $row["#id"] = $agency->value->_id;
  $row["name"] = $agency->value->name;
  if (isset($agency->value->foi_email)) {
  $row["request_email"] = $agency->value->foi_email;
  } else {
  $row["request_email"] = "foi@".GetDomain($agency->value->website);
  //
  }
  if (isset($agency->value->short_name)) {
  $row["short_name"] = $agency->value->short_name;
  } else {
  $out = Array();
  preg_match_all('/[A-Z]/', $agency->value->name, $out);
  $row["short_name"] = implode("",$out[0]);
  }
  $row["notes"] = "";
  $row["publication_scheme"] = $agency->value->infoPublicationSchemeURL;
  $row["home_page"] = $agency->value->website;
  if ($agency->value->orgType == "FMA-DepartmentOfState") {
  $row["tag_string"] = $tag[$agency->value->_id];
  } else {
  $row["tag_string"] = $tag[$agency->value->parentOrg];;
  }
 
  fputcsv($fp, array_values($row));
 
  if (isset($agency->value->foiBodies)) {
  foreach ($agency->value->foiBodies as $foiBody) {
  $row['name'] = $foiBody;
  $row['short_name'] = "";
  fputcsv($fp, array_values($row));
  }
  }
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  die;
  }
  ?>
 
file:b/alaveteli/exportCategories.rb.php (new)
  <?php
  include_once("../include/common.inc.php");
  setlocale(LC_CTYPE, 'C');
  header('Content-Type: text/csv');
  header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
  header('Pragma: no-cache');
  header('Expires: 0');
  echo 'PublicBodyCategories.add(:en, ['.PHP_EOL;
  echo ' "Portfolios",'.PHP_EOL;
  $db = $server->get_db('disclosr-agencies');
 
  try {
  $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
  //print_r($rows);
  foreach ($rows as $row) {
  echo ' [ "'.phrase_to_tag(dept_to_portfolio($row->key)).'","'. dept_to_portfolio($row->key).'","'.dept_to_portfolio($row->key).'" ],'.PHP_EOL;
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
  echo '])';
  ?>
 
file:a/graph.php -> file:b/graph.php
<?php <?php
include_once('include/common.inc.php'); include_once('include/common.inc.php');
//include_header(); //include_header();
?> ?>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script> <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script>
<script src="lib/springy/springy.js"></script> <script src="lib/springy/springy.js"></script>
<script src="lib/springy/springyui.js"></script> <script src="lib/springy/springyui.js"></script>
<script> <script>
var graph = new Graph(); var graph = new Graph();
var nodes = []; var nodes = [];
<?php <?php
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
echo "nodes[\"fedg\"] = graph.newNode({label: \"Federal Government - Commonwealth of Australia\"});" . PHP_EOL; echo "nodes[\"fedg\"] = graph.newNode({label: \"Federal Government - Commonwealth of Australia\"});" . PHP_EOL;
try { try {
$rows = $db->get_view("app", "byCanonicalName", null, true)->rows; $rows = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "nodes[\"{$row->value}\"] = graph.newNode({label: \"{$row->key}\"});" . PHP_EOL; echo "nodes[\"{$row->value->_id}\"] = graph.newNode({label: \"{$row->key}\"});" . PHP_EOL;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
try { try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows; $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "graph.newEdge(nodes[\"fedg\"], nodes['{$row->value}'], {color: '#00A0B0'});" . PHP_EOL; echo "graph.newEdge(nodes[\"fedg\"], nodes['{$row->value}'], {color: '#00A0B0'});" . PHP_EOL;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
try { try {
$rows = $db->get_view("app", "parentOrgs", null, true)->rows; $rows = $db->get_view("app", "parentOrgs", null, true)->rows;
// print_r($rows); // print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "graph.newEdge(nodes[\"{$row->key}\"], nodes['{$row->value}'], {color: '#FFA0B0'});" . PHP_EOL; echo "graph.newEdge(nodes[\"{$row->key}\"], nodes['{$row->value}'], {color: '#FFA0B0'});" . PHP_EOL;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
?> ?>
window.onload = function() { window.onload = function() {
$(document).ready(function() { $(document).ready(function() {
var springy = $('#springydemo').springy({ var springy = $('#springydemo').springy({
graph: graph graph: graph
}); });
}); });
}; };
</script> </script>
   
<canvas id="springydemo" width="1260" height="680" /> <canvas id="springydemo" width="1260" height="680" />
<?php <?php
//include_footer(); //include_footer();
?> ?>
   
   
file:a/include/common.inc.php -> file:b/include/common.inc.php
<?php <?php
   
  date_default_timezone_set("Australia/Sydney");
   
  $basePath = "";
  if (strstr($_SERVER['PHP_SELF'], "alaveteli/")
  || strstr($_SERVER['PHP_SELF'], "myway/")
  || strstr($_SERVER['PHP_SELF'], "lib/")
  || strstr($_SERVER['PHP_SELF'], "geo/")
  || strstr($_SERVER['PHP_SELF'], "include/")
  || strstr($_SERVER['PHP_SELF'], "servicealerts/"))
  $basePath = "../";
   
include_once ('couchdb.inc.php'); include_once ('couchdb.inc.php');
include_once ('template.inc.php'); include_once ('template.inc.php');
   
# Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735 # Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735
   
function object_to_array(stdClass $Class) { function object_to_array(stdClass $Class) {
# Typecast to (array) automatically converts stdClass -> array. # Typecast to (array) automatically converts stdClass -> array.
$Class = (array) $Class; $Class = (array) $Class;
   
# Iterate through the former properties looking for any stdClass properties. # Iterate through the former properties looking for any stdClass properties.
# Recursively apply (array). # Recursively apply (array).
foreach ($Class as $key => $value) { foreach ($Class as $key => $value) {
if (is_object($value) && get_class($value) === 'stdClass') { if (is_object($value) && get_class($value) === 'stdClass') {
$Class[$key] = object_to_array($value); $Class[$key] = object_to_array($value);
} }
} }
return $Class; return $Class;
} }
   
# Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735 # Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735
   
function array_to_object(array $array) { function array_to_object(array $array) {
# Iterate through our array looking for array values. # Iterate through our array looking for array values.
# If found recurvisely call itself. # If found recurvisely call itself.
foreach ($array as $key => $value) { foreach ($array as $key => $value) {
if (is_array($value)) { if (is_array($value)) {
$array[$key] = array_to_object($value); $array[$key] = array_to_object($value);
} }
} }
   
# Typecast to (object) will automatically convert array -> stdClass # Typecast to (object) will automatically convert array -> stdClass
return (object) $array; return (object) $array;
} }
   
  function dept_to_portfolio($deptName) {
  return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName)));
  }
  function phrase_to_tag ($phrase) {
  return str_replace(" ","_",str_replace(",","",strtolower($phrase)));
  }
  function GetDomain($url)
  {
  $nowww = ereg_replace('www\.','',$url);
  $domain = parse_url($nowww);
  if(!empty($domain["host"]))
  {
  return $domain["host"];
  } else
  {
  return $domain["path"];
  }
  }
   
?> ?>
   
   
   
file:a/include/couchdb.inc.php -> file:b/include/couchdb.inc.php
<?php <?php
   
include "schemas/schemas.inc.php"; include $basePath."schemas/schemas.inc.php";
   
  require ($basePath.'couchdb/settee/src/settee.php');
   
function createAgencyDesignDoc() { function createAgencyDesignDoc() {
global $db; global $db;
$obj = new stdClass(); $obj = new stdClass();
$obj->_id = "_design/" . urlencode("app"); $obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript"; $obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->byCanonicalName->map = "function(doc) { $obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') { if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id); emit(doc.name, doc._id);
} }
};"; };";
$obj->views->byDeptStateName->map = "function(doc) { $obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') { if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id); emit(doc.name, doc._id);
} }
};"; };";
$obj->views->parentOrgs->map = "function(doc) { $obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) { if (doc.parentOrg) {
emit(doc._id, doc.parentOrg); emit(doc._id, doc.parentOrg);
} }
};"; };";
$obj->views->byName->map = "function(doc) { $obj->views->byName->map = "function(doc) {
emit(doc.name, doc._id); emit(doc.name, doc._id);
for (name in doc.otherNames) { for (name in doc.otherNames) {
if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) { if (doc.otherNames[name] != '' && doc.otherNames[name] != doc.name) {
emit(doc.otherNames[name], doc._id); emit(doc.otherNames[name], doc._id);
} }
} }
};"; };";
$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }"; $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };'; $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };'; $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
$obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };"; $obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };"; $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
$obj->views->getConflicts->map = "function(doc) { $obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) { if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts)); emit(null, [doc._rev].concat(doc._conflicts));
} }
}"; }";
// http://stackoverflow.com/questions/646628/javascript-startswith // http://stackoverflow.com/questions/646628/javascript-startswith
$obj->views->score->map = 'if(!String.prototype.startsWith){ $obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) { String.prototype.startsWith = function (str) {
return !this.indexOf(str); return !this.indexOf(str);
} }
} }
   
function(doc) { function(doc) {
count = 0; count = 0;
if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
for(var propName in doc) { for(var propName in doc) {
if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) { if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) {
count++ count++
} }
} }
emit(count+doc._id, {id:doc._id, name: doc.name, score:count}); emit(count+doc._id, {id:doc._id, name: doc.name, score:count});
} }
}'; }';
   
// allow safe updates (even if slightly slower due to extra: rev-detection check). // allow safe updates (even if slightly slower due to extra: rev-detection check).
return $db->save($obj, true); return $db->save($obj, true);
} }
   
require ('couchdb/settee/src/settee.php');  
   
if( php_uname('n') == "vanille") { if( php_uname('n') == "vanille") {
   
$server = new SetteeServer('http://192.168.178.21:5984'); $server = new SetteeServer('http://192.168.178.21:5984');
} else } else
if( php_uname('n') == "kyuubey") { if( php_uname('n') == "KYUUBEY") {
   
$server = new SetteeServer('http://192.168.1.8:5984'); $server = new SetteeServer('http://192.168.1.148:5984');
} else { } else {
$server = new SetteeServer('http://127.0.0.1:5984'); $server = new SetteeServer('http://127.0.0.1:5984');
} }
function setteErrorHandler($e) { function setteErrorHandler($e) {
echo $e->getMessage() . "<br>" . PHP_EOL; echo $e->getMessage() . "<br>" . PHP_EOL;
} }
   
?> ?>
   
file:a/schemas/agency.json.php -> file:b/schemas/agency.json.php
<?php <?php
   
$schemas['agency'] = Array( $schemas['agency'] = Array(
"description" => "Representation of government agency and online transparency measures", "description" => "Representation of government agency and online transparency measures",
"type" => "object", "type" => "object",
"properties" => Array( "properties" => Array(
"name" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "name" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
  "short_name" => Array("type" => "string", "required" => false, "x-title"=> "Agency Short Name", "description" => "Agency Name shortened, usually to an acronym"),
"otherNames" => Array("type" => "array", "required" => true, "x-title" => "Agency Past/Other Names", "description" => "Agency Names", "otherNames" => Array("type" => "array", "required" => true, "x-title" => "Agency Past/Other Names", "description" => "Agency Names",
  "items" => Array("type" => "string")),
  "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies", "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous",
"items" => Array("type" => "string")), "items" => Array("type" => "string")),
"orgType"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "orgType"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"parentOrg"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "parentOrg"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"website"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "website"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"abn" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "abn" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"contractListURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "contractListURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"grantsReportingURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "grantsReportingURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"annualReportURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "annualReportURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"consultanciesURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "consultanciesURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"legalExpenditureURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "legalExpenditureURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"recordsListURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "recordsListURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"FOIDocumentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "FOIDocumentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"infoPublicationSchemeURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "infoPublicationSchemeURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"appointmentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "appointmentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"hasRSS"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "hasRSS"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"hasMailingList"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "hasMailingList"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"hasTwitter"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "hasTwitter"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"hasFacebook"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "hasFacebook"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
"hasYouTube"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"), "hasYouTube"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
), ),
/*"org":{"type":"object", /*"org":{"type":"object",
"properties":{ "properties":{
"organizationName":{"type":"string"}, "organizationName":{"type":"string"},
"organizationUnit":{"type":"string"}}, "organizationUnit":{"type":"string"}},
} }
}*/ }*/
); );
?> ?>
   
Unnamed repository; edit this file to name it for gitweb.
GitPHP by Chris Han