| <?php | <?php |
| include_once("../include/common.inc.php"); | require_once '../include/common.inc.php'; |
| require($basePath . 'lib/phpquery/phpQuery/phpQuery.php'); | |
| setlocale(LC_CTYPE, 'C'); | $db = $server->get_db('disclosr-agencies'); |
| $rows = $db->get_view("app", "byName")->rows; | |
| $nametoid = Array(); | |
| $accounts = Array(); | |
| foreach ($rows as $row) { | |
| $nametoid[trim($row->key)] = $row->value; | |
| } | |
| $nameField = "Agency"; | |
| $elevenField = "2011-2012"; | |
| $twelveField = "2012-2013"; | |
| // $request = Requests::get($url); | |
| //echo $url; | |
| // $Data = str_getcsv($request->body, "\n"); //parse the rows | |
| $Data = str_getcsv(file_get_contents("ausbudget.csv"), "\n"); //parse the rows | |
| $headers = Array(); | |
| foreach ($Data as $num => $line) { | |
| $Row = str_getcsv($line, ","); | |
| if ($num == 0) { | |
| $headers = $Row; | |
| print_r($headers); | |
| } else { | |
| if (isset($Row[array_search($nameField, $headers)])) { | |
| $agencyName = $Row[array_search($nameField, $headers)]; | |
| if (!in_array(trim($agencyName), array_keys($nametoid))) { | |
| echo trim($agencyName) . " missing" . PHP_EOL; | |
| } else { | |
| //echo $Row[array_search($nameField, $headers)] . PHP_EOL; | |
| // print_r($Row); | |
| if (!is_array($accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012'])) $accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012'] = Array(); | |
| if (!is_array($accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013'])) $accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013'] = Array(); | |
| $dir = "./ausbudget/"; | if (!isset($accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012']['value'])) $accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012']['value'] = 0; |
| $dhandle = opendir("./ausbudget/"); | if (!isset($accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013']['value'])) $accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013']['value'] = 0; |
| $headers = Array("Table ID", "Portfolio", "Agency", "Program", "Scheme", "2011-2012", "2012-2013", "Difference", "Source"); | $accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012']['value'] += $Row[array_search($elevenField, $headers)]; |
| $accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013']['value'] += $Row[array_search($twelveField, $headers)]; | |
| $fp = fopen('php://output', 'w'); | $accounts[$nametoid[trim($agencyName)]]["budget"]['2011-2012']['source'] = 'http://budgetaus.net/'; |
| if ($fp) { | $accounts[$nametoid[trim($agencyName)]]["budget"]['2012-2013']['source'] = 'http://budgetaus.net/'; |
| header('Content-Type: text/csv; charset=utf-8'); | |
| header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); | |
| header('Pragma: no-cache'); | |
| header('Expires: 0'); | |
| fputcsv($fp, $headers); | |
| if ($dhandle) { | |
| // loop through all of the files | |
| while (false !== ($fname = readdir($dhandle))) { | |
| if (($fname != '.') && ($fname != '..')) { | |
| //echo "$fname <br>"; | |
| $html = phpQuery::newDocumentHTML(file_get_contents($dir . $fname)); | |
| phpQuery::selectDocument($html); | |
| foreach (pq('table')->elements as $table) { | |
| $data = Array(); | |
| ////echo "loltable"; | |
| //echo $table->ownerDocument->saveXML($table); | |
| foreach (pq('tr',$table)->elements as $row) { | |
| //echo "lolrow"; | |
| $rowText = pq($row)->text(); | |
| if (strpos($rowText, "Twitter") === false) { | |
| $key = trim(pq("td:first", $row)->text()); | |
| //echo "<b>$key</b><br>"; | |
| $value = trim(str_replace(pq("td:first", $row)->text(), "", pq("td", $row)->text())); | |
| if ($key == "2011-2012" || $key == "2012-2013") { | |
| $eValue = explode("-", $value); | |
| $value = trim(str_replace(Array("$", ",", "\n"), "", $eValue[0])); | |
| } | |
| //echo "$value <br>"; | |
| $data[$key] = trim(str_replace(Array("\n"), "", $value)); | |
| if ($key == "Source") { | |
| foreach ($headers as $fieldName) { | |
| if (isset($data[$fieldName])) { | |
| $csvrow[] = $data[$fieldName]; | |
| } else { | |
| $csvrow[] = ""; | |
| } | |
| } | |
| fputcsv($fp, array_values($csvrow)); | |
| $data = Array(); | |
| $csvrow = Array(); | |
| } | |
| } | |
| } | |
| } | |
| } | } |
| } else { | |
| echo "error finding any agency" . $line . PHP_EOL; | |
| } | } |
| } | } |
| die; | } |
| //print_r($accounts); | |
| foreach ($accounts as $id => $allvalues) { | |
| echo $id . "<br>" . PHP_EOL; | |
| $doc = object_to_array($db->get($id)); | |
| // print_r($doc); | |
| foreach ($allvalues as $valueType => $values) { | |
| if (!isset($doc['statistics'][$valueType]) || !is_array($doc['statistics'][$valueType])) { | |
| $doc['statistics'][$valueType] = Array(); | |
| } | |
| $doc['statistics'][$valueType] = array_unique(array_merge($doc['statistics'][$valueType], $values)); | |
| } | |
| $db->save($doc); | |
| } | } |
| ?> | ?> |