From: Alex Sadleir Date: Sun, 17 Nov 2013 10:11:20 +0000 Subject: FOI stats importer fixed X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=64737d78435cf1f0d14da3f57e9a3dcbe420e0eb --- FOI stats importer fixed Former-commit-id: 81a6a149848e27565b7a7052d2a7ff4e5aaa9310 --- --- a/admin/importOAICFOIrequests.php +++ b/admin/importOAICFOIrequests.php @@ -17,13 +17,13 @@ if ($row >= 1) { // print_r($data); $name = trim($data[2]); - echo "$name
"; +// echo "$name
"; if ($data[0] != "TOTALS" && $data[0] != "") { if (isset($nametoid[$name])) { $id = $nametoid[$name]; $timePeriod = $data[0] . "-Q" . $data[1]; - echo "$timePeriod
"; +// echo "$timePeriod
"; unset($data[0]); unset($data[1]); unset($data[2]); @@ -38,10 +38,13 @@ $result = Array("source" => "http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/"); foreach ($data as $key => $datum) { if ($datum != 0) { +// tODO prefix header with "FOI" +if (isset($stats[$id][$timePeriod][$key])) $datum += $stats[$id][$timePeriod][$key]; $result[trim($headers[$key])] = $datum; } } $stats[$id][$timePeriod] = $result; +// TODO merge if already exists //print_r($stats); } else { echo "
ERROR NAME MISSING FROM ID LIST

$row" . PHP_EOL; @@ -57,21 +60,24 @@ } fclose($handle); } +echo "all stats loaded successfuly"; foreach ($stats as $id => $stat) { echo $id . "
" . PHP_EOL; - $doc = $db->get($id); + $doc = $db->get($id); echo $doc->name . "
" . PHP_EOL; - print_r($stat); - die(); +// print_r($stat); // print_r($doc); $changed = false; if (!isset($doc->statistics)) { $changed = true; $doc->statistics = Array(); + } else { + $doc->statistics = object_to_array($doc->statistics); } foreach ($stat as $timePeriod => $value) { - if (!isset($doc->statistics->foiRequests->$timePeriod) - || $doc->statistics->foiRequests->$timePeriod != $value) { + if (!isset($doc->statistics["foiRequests"][$timePeriod]) + || $doc->statistics["foiRequests"][$timePeriod] != $value + ) { $changed = true; $doc->statistics["foiRequests"][$timePeriod] = $value; } @@ -81,6 +87,7 @@ } else { echo "not changed" . "
" . PHP_EOL; } +//print_r($doc);die(); } ?> --- a/admin/logo.svg +++ b/admin/logo.svg @@ -1,34 +1,34 @@ - - - - + + + + --- /dev/null +++ b/admin/massdelete.sh @@ -1,1 +1,10 @@ +for line in `curl "http://localhost:5984/disclosr-foidocuments/_design/app/_view/byAgencyID?reduce=false&keys=%5B\"5716ce0aacfe98f7d638b7a66b7f1040\"%5D&limit=600" | xargs -L1`; do +# echo $line + id=`echo $line | grep -Po '_id:.*?[^\\\],' | perl -pe 's/_id://; s/^//; s/,$//'` + rev=`echo $line | grep -Po 'rev:.*?[^\\\],'| perl -pe 's/rev://; s/^//; s/,$//'` + if [ -n "$id" ]; then + echo "curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev" + curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev + fi +done; --- a/admin/refreshDesignDoc.php +++ b/admin/refreshDesignDoc.php @@ -8,21 +8,13 @@ $obj->_id = "_design/" . urlencode("app"); $obj->language = "javascript"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; -$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };"; +$obj->views->byDate->map = "function(doc) { if (doc.title != \"Disclosure Log Updated\") { emit(doc.date, doc); } };"; $obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };"; $obj->views->byDateMonthYear->reduce = "_count"; $obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };"; $obj->views->byAgencyID->reduce = "_count"; -$obj->views->fieldNames->map = ' -function(doc) { -for(var propName in doc) { - emit(propName, doc._id); - } - -}'; -$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { - return values.length; -}'; +$obj->views->fieldNames->map = 'function(doc) { for(var propName in doc) { emit(propName, doc._id); }}'; +$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { return values.length; }'; // allow safe updates (even if slightly slower due to extra: rev-detection check). $foidb->save($obj, true); @@ -120,15 +112,25 @@ } }"; $obj->views->getStatistics->map = -"function(doc) { - if (doc.statistics) { - for (var statisticSet in doc.statistics) { -for (var statisticPeriod in doc.statistics[statisticSet]) { - emit([statisticSet,statisticPeriod], doc.statistics[statisticSet][statisticPeriod]['value']); +" +function (doc) { + if (doc.statistics) { + for (var statisticSet in doc.statistics) { + for (var statisticPeriod in doc.statistics[statisticSet]) { + if (doc.statistics[statisticSet][statisticPeriod]['value']) { + emit([statisticSet, statisticPeriod], doc.statistics[statisticSet][statisticPeriod]['value']); + } else { + for (var statisticSubSet in doc.statistics[statisticSet][statisticPeriod]) { + if (statisticSubSet != 'source' && statisticSubSet != 'value') { + emit([statisticSubSet, statisticPeriod], doc.statistics[statisticSet][statisticPeriod][statisticSubSet]); + } + } + } + } + } + } } -} - } -}"; +"; $obj->views->getStatistics->reduce = '_sum'; // http://stackoverflow.com/questions/646628/javascript-startswith $obj->views->score->map = 'if(!String.prototype.startsWith){ --- a/documents/about.php +++ b/documents/about.php @@ -5,6 +5,7 @@ include_once('../include/common.inc.php'); ?>

About

+Written and managed by Alex Sadleir (maxious [at] lambdacomplex.org) --- a/documents/agency.php +++ b/documents/agency.php @@ -19,7 +19,7 @@
get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows; foreach ($rows as $row) { //print_r($rows); @@ -31,6 +31,12 @@ } else { $rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows; if ($rows) { +function cmp($a, $b) +{ + global $idtoname; + return strcmp($idtoname[$a->key], $idtoname[$b->key]); +} +usort($rows, "cmp"); foreach ($rows as $row) { echo '' . $idtoname[$row->key] . " (" . $row->value . " records)
\n"; } @@ -42,3 +48,4 @@ echo "next page "; include_footer_documents(); ?> + --- a/documents/charts.php +++ b/documents/charts.php @@ -5,18 +5,27 @@ $agenciesdb = $server->get_db('disclosr-agencies'); $idtoname = Array(); +$idtofoirequestssuccessful = Array(); foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) { $idtoname[$row->id] = trim($row->value->name); + $foirequestssuccessful = 0; +if(isset($row->value->statistics->foiRequests)) { + foreach ($row->value->statistics->foiRequests as $statperiod) { + $statperiod=object_to_array($statperiod); + if (isset($statperiod["Requests for other information granted in full"])) $foirequestssuccessful += $statperiod["Requests for other information granted in full"]; + if (isset($statperiod["Requests for other information granted in part"])) $foirequestssuccessful += $statperiod["Requests for other information granted in part"]; + } +} + $idtofoirequestssuccessful[$row->id] =$foirequestssuccessful; } $foidocsdb = $server->get_db('disclosr-foidocuments'); - ?>

About

Charts