fix scraping
fix scraping


Former-commit-id: c96cc5c23e3497cb03991f1ee4e2990548817cf3

[submodule "couchdb/couchdb-lucene"] [submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git url = https://github.com/rnewson/couchdb-lucene.git
[submodule "couchdb/settee"]  
path = couchdb/settee  
url = https://github.com/inadarei/settee.git  
[submodule "lib/php-diff"] [submodule "lib/php-diff"]
path = lib/php-diff path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git url = https://github.com/chrisboulton/php-diff.git
[submodule "lib/Requests"] [submodule "lib/Requests"]
path = lib/Requests path = lib/Requests
url = https://github.com/rmccue/Requests.git url = https://github.com/rmccue/Requests.git
[submodule "js/flotr2"] [submodule "js/flotr2"]
path = js/flotr2 path = js/flotr2
url = https://github.com/HumbleSoftware/Flotr2.git url = https://github.com/HumbleSoftware/Flotr2.git
[submodule "lib/phpquery"] [submodule "lib/phpquery"]
path = lib/phpquery path = lib/phpquery
url = https://github.com/TobiaszCudnik/phpquery.git url = https://github.com/TobiaszCudnik/phpquery.git
[submodule "js/sigma"] [submodule "js/sigma"]
path = js/sigma path = js/sigma
url = https://github.com/jacomyal/sigma.js.git url = https://github.com/jacomyal/sigma.js.git
[submodule "js/bubbletree"] [submodule "js/bubbletree"]
path = js/bubbletree path = js/bubbletree
url = https://github.com/okfn/bubbletree.git url = https://github.com/okfn/bubbletree.git
[submodule "lib/querypath"] [submodule "lib/querypath"]
path = lib/querypath path = lib/querypath
url = https://github.com/technosophos/querypath.git url = https://github.com/technosophos/querypath.git
[submodule "lib/amon-php"] [submodule "lib/amon-php"]
path = lib/amon-php path = lib/amon-php
url = https://github.com/martinrusev/amon-php.git url = https://github.com/martinrusev/amon-php.git
[submodule "documents/lib/parsedatetime"] [submodule "documents/lib/parsedatetime"]
path = documents/lib/parsedatetime path = documents/lib/parsedatetime
url = git://github.com/bear/parsedatetime.git url = git://github.com/bear/parsedatetime.git
  [submodule "lib/FeedWriter"]
  path = lib/FeedWriter
  url = https://github.com/mibe/FeedWriter
   
<?php <?php
   
include_once("../include/common.inc.php"); include_once("../include/common.inc.php");
   
$format = "csv"; $format = "csv";
//$format = "json"; //$format = "json";
if (isset($_REQUEST['format'])) $format = $_REQUEST['format']; if (isset($_REQUEST['format']))
  $format = $_REQUEST['format'];
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
if ($format == "csv") { if ($format == "csv") {
$headers = Array("name"); $headers = Array("name");
} else { } else {
$headers = Array(); $headers = Array();
} }
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
try { try {
$rows = $db->get_view("app", "all", null, true)->rows; $rows = $db->get_view("app", "all", null, true)->rows;
   
$dataValues = Array(); $dataValues = Array();
foreach ($rows as $row) { foreach ($rows as $row) {
if (isset($row->value->statistics->employees)) { if (isset($row->value->statistics->employees)) {
   
$headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees)))); $headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees))));
   
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
   
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $db) { if ($fp && $db) {
if ($format == "csv") { if ($format == "csv") {
header('Content-Type: text/csv; charset=utf-8'); header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.employeestats.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.employeestats.' . date("c") . '.csv"');
} }
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
if ($format == "csv") { if ($format == "csv") {
fputcsv($fp, $headers); fputcsv($fp, $headers);
} else if ($format == "json") { } else if ($format == "json") {
echo '{ echo '{
"labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL; "labels" : ["' . implode('","', $headers) . '"],' . PHP_EOL;
} }
try { try {
$agencies = $db->get_view("app", "all", null, true)->rows; $agencies = $db->get_view("app", "all", null, true)->rows;
//print_r($agencies); //print_r($agencies);
$first = true; $first = true;
if ($format == "json") { if ($format == "json") {
echo '"data" : ['.PHP_EOL; echo '"data" : [' . PHP_EOL;
   
} }
foreach ($agencies as $agency) { foreach ($agencies as $agency) {
   
if (isset($agency->value->statistics->employees)) { if (isset($agency->value->statistics->employees)) {
$row = Array(); $row = Array();
$agencyEmployeesArray = object_to_array($agency->value->statistics->employees); $agencyEmployeesArray = object_to_array($agency->value->statistics->employees);
foreach ($headers as $i => $fieldName) { foreach ($headers as $i => $fieldName) {
  if ($format == "csv") {
  if (isset($agencyEmployeesArray[$fieldName])) {
  $row[] = $agencyEmployeesArray[$fieldName]["value"] ;
  } else if ($i == 0) {
  $row[] = $agency->value->name;
  } else {
  $row[] = 0;
  }
  } else if ($format == "json") {
if (isset($agencyEmployeesArray[$fieldName])) { if (isset($agencyEmployeesArray[$fieldName])) {
$row[] = '['.$i.','.$agencyEmployeesArray[$fieldName]["value"].']'; $row[] = '[' . $i . ',' . $agencyEmployeesArray[$fieldName]["value"] . ']';
} else { } else {
$row[] = '['.$i.',0]'; $row[] = '[' . $i . ',0]';
} }
  }
} }
if ($format == "csv") { if ($format == "csv") {
fputcsv($fp, array_values($row)); fputcsv($fp, array_values($row));
} else if ($format == "json") { } else if ($format == "json") {
if (!$first) echo ","; if (!$first)
echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL; echo ",";
  echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "' . $agency->value->name . '", "lines" : { "show" : true }, "points" : { "show" : true }}' . P