projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Add validation script
Add validation script


Former-commit-id: 8f0da9f23f1e65c57b3e158ef358630a33f10141

file:a/.gitmodules -> file:b/.gitmodules
[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"] [submodule "couchdb/settee"]
path = couchdb/settee path = couchdb/settee
url = https://github.com/inadarei/settee.git url = https://github.com/inadarei/settee.git
[submodule "lib/springy"] [submodule "lib/springy"]
path = lib/springy path = lib/springy
url = https://github.com/dhotson/springy.git url = https://github.com/dhotson/springy.git
[submodule "lib/php-diff"] [submodule "lib/php-diff"]
path = lib/php-diff path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git url = https://github.com/chrisboulton/php-diff.git
[submodule "lib/Requests"] [submodule "lib/Requests"]
path = lib/Requests path = lib/Requests
url = https://github.com/rmccue/Requests.git url = https://github.com/rmccue/Requests.git
[submodule "javascripts/flotr2"] [submodule "javascripts/flotr2"]
path = javascripts/flotr2 path = javascripts/flotr2
url = https://github.com/HumbleSoftware/Flotr2.git url = https://github.com/HumbleSoftware/Flotr2.git
  [submodule "lib/phpquery"]
  path = lib/phpquery
  url = https://github.com/TobiaszCudnik/phpquery.git
   
file:b/admin/exportEmployees.csv.php (new)
  <?php
 
  include_once("../include/common.inc.php");
 
  $format = "csv";
  //$format = "json";
  if (isset($_REQUEST['format'])) $format = $_REQUEST['format'];
 
  setlocale(LC_CTYPE, 'C');
  if ($format == "csv") {
  $headers = Array("name");
  } else {
  $headers = Array();
  }
 
  $db = $server->get_db('disclosr-agencies');
  try {
  $rows = $db->get_view("app", "all", null, true)->rows;
 
  $dataValues = Array();
  foreach ($rows as $row) {
  if (isset($row->value->statistics->employees)) {
 
  $headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees))));
 
  }
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  $fp = fopen('php://output', 'w');
  if ($fp && $db) {
  if ($format == "csv") {
  header('Content-Type: text/csv; charset=utf-8');
  header('Content-Disposition: attachment; filename="export.employeestats.' . date("c") . '.csv"');
  }
  header('Pragma: no-cache');
  header('Expires: 0');
  if ($format == "csv") {
  fputcsv($fp, $headers);
  } else if ($format == "json") {
  echo '{
  "labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL;
  }
  try {
  $agencies = $db->get_view("app", "all", null, true)->rows;
  //print_r($agencies);
  $first = true;
  if ($format == "json") {
  echo '"data" : ['.PHP_EOL;
 
  }
  foreach ($agencies as $agency) {
 
  if (isset($agency->value->statistics->employees)) {
  $row = Array();
  $agencyEmployeesArray = object_to_array($agency->value->statistics->employees);
  foreach ($headers as $i => $fieldName) {
  if (isset($agencyEmployeesArray[$fieldName])) {
  $row[] = '['.$i.','.$agencyEmployeesArray[$fieldName]["value"].']';
  } else {
  $row[] = '['.$i.',0]';
  }
  }
  if ($format == "csv") {
  fputcsv($fp, array_values($row));
  } else if ($format == "json") {
  if (!$first) echo ",";
  echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL;
  $first = false;
  }
  }
  }
  if ($format == "json") {
  echo ']
  }'.PHP_EOL;
 
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  die;
  }
  ?>
 
file:b/admin/exportScore.csv.php (new)
  <?php
 
  include_once("../include/common.inc.php");
 
  $db = $server->get_db('disclosr-agencies');
  $format = "csv";
  //$format = "json";
  if (isset($_REQUEST['format'])) $format = $_REQUEST['format'];
 
  setlocale(LC_CTYPE, 'C');
 
  $headers = Array();
 
  $fp = fopen('php://output', 'w');
  if ($fp && $db) {
  if ($format == "csv") {
  header('Content-Type: text/csv; charset=utf-8');
  header('Content-Disposition: attachment; filename="export.score.' . date("c") . '.csv"');
  }
  header('Pragma: no-cache');
  header('Expires: 0');
 
  try {
  $agencies = $db->get_view("score", "score", null, true)->rows;
  //print_r($agencies);
  $first = true;
  if ($format == "json") {
  echo '"data" : ['.PHP_EOL;
 
  }
  foreach ($agencies as $agency) {
  $agencyArray = object_to_array($agency->value);
  if ($first) {
  $headers = array_keys($agencyArray);
  if ($format == "csv") {
  fputcsv($fp, $headers);
  } else if ($format == "json") {
  echo '{
  "labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL;
  }
  }
  $row = Array();
 
  foreach ($headers as $i => $fieldName) {
  if (isset($agencyArray[$fieldName])) {
  $row[] = $agencyArray[$fieldName];
  } else {
  $row[] = '';
  }
  }
  if ($format == "csv") {
  fputcsv($fp, array_values($row));
  } else if ($format == "json") {
  if (!$first) echo ",";
  echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL;
 
  }
  $first = false;
  }
 
  if ($format == "json") {
  echo ']
  }'.PHP_EOL;
 
  }
  } catch (SetteeRestClientException $e) {
  setteErrorHandler($e);
  }
 
  die;
  }
  ?>
 
file:b/admin/importAustraliaGovAuGov2.php (new)
  <?php
 
  require_once '../include/common.inc.php';
 
  $db = $server->get_db('disclosr-agencies');
  $rows = $db->get_view("app", "byName")->rows;
  $nametoid = Array();
  $accounts = Array();
  foreach ($rows as $row) {
  $nametoid[trim($row->key)] = $row->value;
  }
 
  function extractCSVAccounts($url, $nameField, $accountField, $filter) {
  global $accounts, $nametoid;
  $request = Requests::get($url);
  $Data = str_getcsv($request->body, "\n"); //parse the rows
  $headers = Array();
  foreach ($Data as $num => $line) {
  $Row = str_getcsv($line, ",");
  if ($num == 0) {
 
  } else if ($num == 1) {
  $headers = $Row;
  //print_r($headers);
  } else {
  if (isset($Row[array_search($nameField, $headers)])) {
  $agencyName = $Row[array_search($nameField, $headers)];
  if (!$filter || $Row[array_search("State", $headers)] == "NAT") {
  if (!in_array(trim($agencyName), array_keys($nametoid))) {
  echo "$agencyName missing" . PHP_EOL;
  } else {
  // echo $Row[array_search($nameField, $headers)] . PHP_EOL;
  }
  }
  } else {
  //echo "error finding agency" . $line . PHP_EOL;
  }
  }
  }
  }
 
  // http://agimo.govspace.gov.au/page/gov2register/
  // twitter
  //extractCSVAccounts("https://docs.google.com/spreadsheet/pub?key=0Ap1exl80wB8OdHNKVmQ5RVlvQWpibDAxNHkzcU1nV2c&single=true&gid=0&output=csv", "Agency/Body/Event", "", true);
  // RSS
  // https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGJxandJREhLSGlWWUZfZ2xKOTNHZ0E&output=csv
  // facebook
  extractCSVAccounts("https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGtjcW9vOXdyZ3pOV21vQU51VmhzQnc&single=true&gid=0&output=csv","Agency","Name");
 
  /*
  * http://australia.gov.au/news-and-media/media-release-rss-feeds
  * http://australia.gov.au/news-and-media/social-media/blogs
  * http://australia.gov.au/news-and-media/social-media/twitter
  * http://australia.gov.au/news-and-media/social-media/facebook
  * http://australia.gov.au/news-and-media/social-media/youtube
  * http://australia.gov.au/news-and-media/social-media/flickr
  * http://australia.gov.au/news-and-media/social-media/apps http://www.harmony.gov.au/get-involved/app-downloads.htm http://www.em.gov.au/Resources/Pages/Before-the-Storm-phone-game.aspx
  * http://australia.gov.au/news-and-media/social-media/podcasts
  */
  ?>
 
file:a/admin/importDirectoryUUIDs.php -> file:b/admin/importDirectoryUUIDs.php
<?php <?php
   
// http://gold.gov.au/reports/department/index.xml // http://gold.gov.au/reports/department/index.xml
require_once '../include/common.inc.php'; require_once '../include/common.inc.php';
try { $db = $server->get_db('disclosr-agencies');
$server->create_db('disclosr-agencies'); $rows = $db->get_view("app", "byName")->rows;
} catch (SetteeRestClientException $e) { $nametoid = Array();
setteErrorHandler($e); $sums = Array();
  foreach ($rows as $row) {
  $nametoid[trim($row->key)] = $row->value;
} }
$db = $server->get_db('disclosr-agencies');  
createAgencyDesignDoc();  
   
  if (file_exists('index.xml')) {
  $xml = simplexml_load_file('index.xml');
   
  foreach ($xml as $agency) {
  $names = Array();
  $names[] = "".$agency->name;
   
  if (isset($agency->shortName)) {
  $names[] = "".$agency->shortName;
  }
  foreach ($names as $name) {
  if (!in_array($name, array_keys($nametoid))) {
  echo "$name missing".PHP_EOL;
  print_r($names);
  echo $agency->dn;
  echo PHP_EOL;
  echo PHP_EOL;
  }
  }
  }
  } else {
  exit('Failed to open test.xml.');
  }
?> ?>
   
file:a/admin/importGov2RegisterRSSFacebookTwitter.php -> file:b/admin/importGov2RegisterRSSFacebookTwitter.php
<
<?php <?php
   
require_once '../include/common.inc.php'; require_once '../include/common.inc.php';
try { require($basePath . 'lib/phpquery/phpQuery/phpQuery.php');
$server->create_db('disclosr-agencies');  
} catch (SetteeRestClientException $e) { $db = $server->get_db('disclosr-agencies');
setteErrorHandler($e); $rows = $db->get_view("app", "byName")->rows;
  $nametoid = Array();
  $accounts = Array();
  foreach ($rows as $row) {
  $nametoid[trim($row->key)] = $row->value;
} }
$db = $server->get_db('disclosr-agencies');  
createAgencyDesignDoc();  
   
// twitter https://docs.google.com/spreadsheet/fm?id=tsJVd9EYoAjbl014y3qMgWg.03918275400592898296.8568379511161083736&hl=en&fmcmd=5&gid=0 function extractHTMLAccounts($url, $accountType) {
// RSS https://docs.google.com/spreadsheet/fm?id=tbqjwIDHKHiVYF_glJ93GgA.03918275400592898296.8789688748524615194&authkey=CJDP-uQG&hl=en_GB&fmcmd=5&gid=0 global $accounts, $nametoid;
// facebook https://docs.google.com/spreadsheet/fm?id=tkcqoo9wrgzNWmoANuVhsBw.03918275400592898296.3040387705062056060&authkey=CKzl7r0I&hl=en_GB&fmcmd=5&gid=0 $request = Requests::get($url);
  $doc = phpQuery::newDocumentHTML($request->body);
  phpQuery::selectDocument($doc);
  foreach (pq('tr')->elements as $tr) {
  //echo $tr->nodeValue.PHP_EOL;
  $agency = "";
  $url = "";
  foreach ($tr->childNodes as $td) {
  $class = $td->getAttribute("class");
  //echo "cccc $class ".$td->nodeValue.PHP_EOL;
  if ($class == "s11" || $class == "s10" || $class == "s7") {
  $agency = $td->nodeValue;
  } else if ($class == "s6" || $class == "s9") {
  $url = $td->nodeValue;
  foreach ($td->childNodes as $a) {
  $href = $a->getAttribute("href");
  if ($href != "") {
  $url = $href;
  }
  }
  }
  }
  if ($agency != "" && $url != "") {
  if (!in_array(trim($agency), array_keys($nametoid))) {
  echo trim($agency) . " missing" . PHP_EOL;
  } else {
  // echo $agency." = ".$url.PHP_EOL;
  $accounts[$nametoid[trim($agency)]][$accountType][] = $url;
  }
  }
  }
  }
   
  function extractCSVAccounts($url, $accountType, $nameField, $accountField, $filter) {
  global $accounts, $nametoid;
  $request = Requests::get($url);
  $Data = str_getcsv($request->body, "\n"); //parse the rows
  $headers = Array();
  foreach ($Data as $num => $line) {
  $Row = str_getcsv($line, ",", '"');
  if ($num == 0) {
   
  } else if ($num == 1) {
  $headers = $Row;
  //print_r($headers);
  } else {
  if (isset($Row[array_search($nameField, $headers)])) {
  $agencyName = $Row[array_search($nameField, $headers)];
  if (!$filter || $Row[array_search("State", $headers)] == "NAT") {
  if (!in_array(trim($agencyName), array_keys($nametoid))) {
  echo trim($agencyName) . " missing" . PHP_EOL;
  } else {
  // echo $Row[array_search($nameField, $headers)] . PHP_EOL;
  $accounts[$nametoid[trim($agencyName)]][$accountType][] = $Row[array_search($accountField, $headers)];
  }
  }
  } else {
  //echo "error finding agency" . $line . PHP_EOL;
  }