projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Make apsc stats importer nondestructive
Make apsc stats importer nondestructive


Former-commit-id: 1eae8d1c89fabed230ce5d3f02595c29a41b7f5e

file:a/.gitmodules -> file:b/.gitmodules
[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"] [submodule "couchdb/settee"]
path = couchdb/settee path = couchdb/settee
url = https://github.com/inadarei/settee.git url = https://github.com/inadarei/settee.git
[submodule "lib/springy"]  
path = lib/springy  
url = https://github.com/dhotson/springy.git  
[submodule "lib/php-diff"] [submodule "lib/php-diff"]
path = lib/php-diff path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git url = https://github.com/chrisboulton/php-diff.git
[submodule "lib/Requests"] [submodule "lib/Requests"]
path = lib/Requests path = lib/Requests
url = https://github.com/rmccue/Requests.git url = https://github.com/rmccue/Requests.git
[submodule "javascripts/flotr2"] [submodule "javascripts/flotr2"]
path = javascripts/flotr2 path = javascripts/flotr2
url = https://github.com/HumbleSoftware/Flotr2.git url = https://github.com/HumbleSoftware/Flotr2.git
  [submodule "lib/phpquery"]
  path = lib/phpquery
  url = https://github.com/TobiaszCudnik/phpquery.git
  [submodule "javascripts/sigma"]
  path = javascripts/sigma
  url = https://github.com/jacomyal/sigma.js.git
  [submodule "javascripts/bubbletree"]
  path = javascripts/bubbletree
  url = https://github.com/okfn/bubbletree.git
   
file:a/admin/exportEmployees.csv.php -> file:b/admin/exportEmployees.csv.php
<?php <?php
   
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
   
$format = "csv"; $format = "csv";
//$format = "json"; //$format = "json";
if (isset($_REQUEST['format'])) $format = $_REQUEST['format']; if (isset($_REQUEST['format'])) $format = $_REQUEST['format'];
   
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
if ($format == "csv") { if ($format == "csv") {
$headers = Array("name"); $headers = Array("name");
} else { } else {
$headers = Array(); $headers = Array();
} }
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
try { try {
$rows = $db->get_view("app", "all", null, true)->rows; $rows = $db->get_view("app", "all", null, true)->rows;
   
$dataValues = Array(); $dataValues = Array();
foreach ($rows as $row) { foreach ($rows as $row) {
if (isset($row->value->statistics->employees)) { if (isset($row->value->statistics->employees)) {
   
$headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees)))); $headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees))));
   
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $db) { if ($fp && $db) {
if ($format == "csv") { if ($format == "csv") {
header('Content-Type: text/csv; charset=utf-8'); header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.employeestats.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.employeestats.' . date("c") . '.csv"');
} }
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
if ($format == "csv") { if ($format == "csv") {
fputcsv($fp, $headers); fputcsv($fp, $headers);
} else if ($format == "json") { } else if ($format == "json") {
echo '{ echo '{
"labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL; "labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL;
} }
try { try {
$agencies = $db->get_view("app", "all", null, true)->rows; $agencies = $db->get_view("app", "all", null, true)->rows;
//print_r($agencies); //print_r($agencies);
$first = true; $first = true;
if ($format == "json") { if ($format == "json") {
echo '"data" : ['.PHP_EOL; echo '"data" : ['.PHP_EOL;
} }
foreach ($agencies as $agency) { foreach ($agencies as $agency) {
   
if (isset($agency->value->statistics->employees)) { if (isset($agency->value->statistics->employees)) {
$row = Array(); $row = Array();
$agencyEmployeesArray = object_to_array($agency->value->statistics->employees); $agencyEmployeesArray = object_to_array($agency->value->statistics->employees);
foreach ($headers as $i => $fieldName) { foreach ($headers as $i => $fieldName) {
if (isset($agencyEmployeesArray[$fieldName])) { if (isset($agencyEmployeesArray[$fieldName])) {
$row[] = '['.$i.','.$agencyEmployeesArray[$fieldName]["value"].']'; $row[] = '['.$i.','.$agencyEmployeesArray[$fieldName]["value"].']';
} else { } else {
$row[] = '['.$i.',0]'; $row[] = '['.$i.',0]';
} }
} }
if ($format == "csv") { if ($format == "csv") {
fputcsv($fp, array_values($row)); fputcsv($fp, array_values($row));
} else if ($format == "json") { } else if ($format == "json") {
if (!$first) echo ","; if (!$first) echo ",";
echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL; echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL;
$first = false; $first = false;
} }
} }
} }
if ($format == "json") { if ($format == "json") {
echo '] echo ']
}'.PHP_EOL; }'.PHP_EOL;
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
die; die;
} }
?> ?>
   
file:b/admin/exportScore.csv.php (new)
  <?php
 
  include_once("../include/common.inc.php");
 
  $db = $server->get_db('disclosr-agencies');
  $format = "csv";
  //$format = "json";
  if (isset($_REQUEST['format'])) $format = $_REQUEST['format'];
 
  setlocale(LC_CTYPE, 'C');
 
  $headers = Array();
 
  $fp = fopen('php://output', 'w');
  if ($fp && $db) {
  if ($format == "csv") {
  header('Content-Type: text/csv; charset=utf-8');
  header('Content-Disposition: attachment; filename="export.score.' . date("c") . '.csv"');
  }
  header('Pragma: no-cache');
  header('Expires: 0');
 
  try {
  $agencies = $db->get_view("score", "score", null, true)->rows;
  //print_r($agencies);
  $first = true;
  if ($format == "json") {
  echo '"data" : ['.PHP_EOL;
 
  }
  foreach ($agencies as $agency) {
  $agencyArray = object_to_array($agency->value);
  if ($first) {
  $headers = array_keys($agencyArray);
  if ($format == "csv") {
  fputcsv($fp, $headers);
  } else if ($format == "json") {
  echo '{
  "labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL;
  }
  }
  $row = Array();
 
  foreach ($headers as $i => $fieldName) {
  if (isset($agencyArray[$fieldName])) {
  $row[] = $agencyArray[$fieldName];
  } else {
  $row[] = '';
  }
  }
  if ($format == "csv") {
  fputcsv($fp, array_values($row));
  } else if ($format == "json") {
  if (!$first) echo ",";
  echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL;
 
  }
  $first = false;
  }
 
  if ($format == "json") {
  echo ']
  }'.PHP_EOL;
 
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  die;
  }
  ?>
 
file:a/admin/importAPSCEmployees.php -> file:b/admin/importAPSCEmployees.php
<?php <?php
   
require_once '../include/common.inc.php'; require_once '../include/common.inc.php';
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
$rows = $db->get_view("app", "byName")->rows; $rows = $db->get_view("app", "byName")->rows;
$nametoid = Array(); $nametoid = Array();
$sums = Array(); $sums = Array();
foreach ($rows as $row) { foreach ($rows as $row) {
$nametoid[trim($row->key)] = $row->value; $nametoid[trim($row->key)] = $row->value;
} }
$employeeCSVs = Array("2002-2003" => "0203apsemployees.csv", $employeeCSVs = Array("2002-2003" => "0203apsemployees.csv",
"2003-2004" => "0304apsemployees.csv", "2003-2004" => "0304apsemployees.csv",
"2004-2005" => "0405apsemployees.csv", "2004-2005" => "0405apsemployees.csv",
"2005-2006" => "0506apsemployees.csv", "2005-2006" => "0506apsemployees.csv",
"2006-2007" => "0607apsemployees.csv", "2006-2007" => "0607apsemployees.csv",
"2007-2008" => "0708apsemployees.csv", "2007-2008" => "0708apsemployees.csv",
"2008-2009" => "0809apsemployees.csv", "2008-2009" => "0809apsemployees.csv",
"2009-2010" => "0910apsemployees.csv", "2009-2010" => "0910apsemployees.csv",
"2010-2011" => "1011apsemployees.csv" "2010-2011" => "1011apsemployees.csv"
); );
foreach ($employeeCSVs as $timePeriod => $employeeCSV) { foreach ($employeeCSVs as $timePeriod => $employeeCSV) {
echo $employeeCSV . "<br>" . PHP_EOL; echo $employeeCSV . "<br>" . PHP_EOL;
$row = 1; $row = 1;
if (($handle = fopen($employeeCSV, "r")) !== FALSE) { if (($handle = fopen($employeeCSV, "r")) !== FALSE) {
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
//print_r($data); //print_r($data);
$name = trim($data[0]); $name = trim($data[0]);
if (isset($nametoid[$name])) { if (isset($nametoid[$name])) {
$id = $nametoid[$name]; $id = $nametoid[$name];
//echo $id . "<br>" . PHP_EOL; //echo $id . "<br>" . PHP_EOL;
@$sums[$id][$timePeriod] += $data[1]; @$sums[$id][$timePeriod] += $data[1];
} else { } else {
echo "<br>ERROR NAME MISSING FROM ID LIST<br><bR>" . PHP_EOL; echo "<br>ERROR NAME MISSING FROM ID LIST<br><bR>" . PHP_EOL;
   
die(); die();
   
} }
} }
fclose($handle); fclose($handle);
} }
} }
foreach ($sums as $id => $sum) { foreach ($sums as $id => $sum) {
echo $id. "<br>" . PHP_EOL; echo $id . "<br>" . PHP_EOL;
$doc = $db->get($id); $doc = $db->get($id);
// print_r($doc); echo $doc->name . "<br>" . PHP_EOL;
if (isset($doc->statistics)) $doc->statistics = Array(); // print_r($doc);
  $changed = false;
  if (!isset($doc->statistics)) {
  $changed = true;
  $doc->statistics = Array();
  }
foreach ($sum as $timePeriod => $value) { foreach ($sum as $timePeriod => $value) {
$doc->statistics["employees"][$timePeriod] = Array("value"=>$value, "source"=>"http://apsc.gov.au/stateoftheservice/"); if (!isset($doc->statistics->employees->$timePeriod->value)
  || $doc->statistics->employees->$timePeriod->value != $value) {
  $changed = true;
  $doc->statistics["employees"][$timePeriod] = Array("value" => $value, "source" => "http://apsc.gov.au/stateoftheservice/");
  }
} }
$db->save($doc); if ($changed) {
  $db->save($doc);
  } else {
  echo "not changed" . "<br>" . PHP_EOL;
  }
} }
// employees: timeperiod, source = apsc state of service, value // employees: timeperiod, source = apsc state of service, value
?> ?>
   
file:b/admin/importAustraliaGovAuGov2.php (new)
  <?php
 
  require_once '../include/common.inc.php';
 
  $db = $server->get_db('disclosr-agencies');
  $rows = $db->get_view("app", "byName")->rows;
  $nametoid = Array();
  $accounts = Array();
  foreach ($rows as $row) {
  $nametoid[trim($row->key)] = $row->value;
  }
 
  function extractCSVAccounts($url, $nameField, $accountField, $filter) {
  global $accounts, $nametoid;
  $request = Requests::get($url);
  $Data = str_getcsv($request->body, "\n"); //parse the rows
  $headers = Array();
  foreach ($Data as $num => $line) {
  $Row = str_getcsv($line, ",");
  if ($num == 0) {
 
  } else if ($num == 1) {
  $headers = $Row;
  //print_r($headers);
  } else {
  if (isset($Row[array_search($nameField, $headers)])) {
  $agencyName = $Row[array_search($nameField, $headers)];
  if (!$filter || $Row[array_search("State", $headers)] == "NAT") {
  if (!in_array(trim($agencyName), array_keys($nametoid))) {
  echo "$agencyName missing" . PHP_EOL;
  } else {
  // echo $Row[array_search($nameField, $headers)] . PHP_EOL;
  }
  }
  } else {
  //echo "error finding agency" . $line . PHP_EOL;
  }
  }
  }
  }
 
  // http://agimo.govspace.gov.au/page/gov2register/
  // twitter
  //extractCSVAccounts("https://docs.google.com/spreadsheet/pub?key=0Ap1exl80wB8OdHNKVmQ5RVlvQWpibDAxNHkzcU1nV2c&single=true&gid=0&output=csv", "Agency/Body/Event", "", true);
  // RSS
  // https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGJxandJREhLSGlWWUZfZ2xKOTNHZ0E&output=csv
  // facebook
  extractCSVAccounts("https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGtjcW9vOXdyZ3pOV21vQU51VmhzQnc&single=true&gid=0&output=csv","Agency","Name");
 
  /*
  * http://australia.gov.au/news-and-media/media-release-rss-feeds
  * http://australia.gov.au/news-and-media/social-media/blogs
  * http://australia.gov.au/news-and-media/social-media/twitter
  * http://australia.gov.au/news-and-media/social-media/facebook
  * http://australia.gov.au/news-and-media/social-media/youtube
  * http://australia.gov.au/news-and-media/social-media/flickr
  * http://australia.gov.au/news-and-media/social-media/apps http://www.harmony.gov.au/get-involved/app-downloads.htm http://www.em.gov.au/Resources/Pages/Before-the-Storm-phone-game.aspx
  * http://australia.gov.au/news-and-media/social-media/podcasts
  */
  ?>