From: Maxious Date: Thu, 29 Nov 2012 12:14:23 +0000 Subject: mo scrapers and apsc2012 imoort X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=992a7a9dd6bd073652eb6d4cdf63cd9cce9a815e --- mo scrapers and apsc2012 imoort Former-commit-id: 65baeca74e6e972c9a6b51dd8b21387e9b017d92 --- --- a/admin/exportEmployees.csv.php +++ b/admin/exportEmployees.csv.php @@ -4,7 +4,8 @@ $format = "csv"; //$format = "json"; -if (isset($_REQUEST['format'])) $format = $_REQUEST['format']; +if (isset($_REQUEST['format'])) + $format = $_REQUEST['format']; setlocale(LC_CTYPE, 'C'); if ($format == "csv") { $headers = Array("name"); @@ -21,7 +22,6 @@ if (isset($row->value->statistics->employees)) { $headers = array_unique(array_merge($headers, array_keys(object_to_array($row->value->statistics->employees)))); - } } } catch (SetteeRestClientException $e) { @@ -40,15 +40,14 @@ fputcsv($fp, $headers); } else if ($format == "json") { echo '{ - "labels" : ["' . implode('","', $headers) . '"],'.PHP_EOL; + "labels" : ["' . implode('","', $headers) . '"],' . PHP_EOL; } try { $agencies = $db->get_view("app", "all", null, true)->rows; //print_r($agencies); $first = true; if ($format == "json") { - echo '"data" : ['.PHP_EOL; - + echo '"data" : [' . PHP_EOL; } foreach ($agencies as $agency) { @@ -56,25 +55,35 @@ $row = Array(); $agencyEmployeesArray = object_to_array($agency->value->statistics->employees); foreach ($headers as $i => $fieldName) { + if ($format == "csv") { + if (isset($agencyEmployeesArray[$fieldName])) { + $row[] = $agencyEmployeesArray[$fieldName]["value"] ; + } else if ($i == 0) { + $row[] = $agency->value->name; + } else { + $row[] = 0; + } + } else if ($format == "json") { if (isset($agencyEmployeesArray[$fieldName])) { - $row[] = '['.$i.','.$agencyEmployeesArray[$fieldName]["value"].']'; + $row[] = '[' . $i . ',' . $agencyEmployeesArray[$fieldName]["value"] . ']'; } else { - $row[] = '['.$i.',0]'; + $row[] = '[' . $i . ',0]'; } + } } if ($format == "csv") { fputcsv($fp, array_values($row)); } else if ($format == "json") { - if (!$first) echo ","; - echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "'.$agency->value->name.'", "lines" : { "show" : true }, "points" : { "show" : true }}'.PHP_EOL; + if (!$first) + echo ","; + echo '{"data" : [' . implode(",", array_values($row)) . '], "label": "' . $agency->value->name . '", "lines" : { "show" : true }, "points" : { "show" : true }}' . PHP_EOL; $first = false; } } } if ($format == "json") { - echo '] - }'.PHP_EOL; - + echo '] + }' . PHP_EOL; } } catch (SetteeRestClientException $e) { setteErrorHandler($e); --- a/admin/importAPSCEmployees.php +++ b/admin/importAPSCEmployees.php @@ -47,13 +47,17 @@ $changed = false; if (!isset($doc->statistics)) { $changed = true; - $doc->statistics = Array(); + $doc->statistics = new stdClass(); + } + if (!isset($doc->statistics->employees)) { + $changed = true; + $doc->statistics->employees = new stdClass(); } foreach ($sum as $timePeriod => $value) { if (!isset($doc->statistics->employees->$timePeriod->value) || $doc->statistics->employees->$timePeriod->value != $value) { $changed = true; - $doc->statistics["employees"][$timePeriod] = Array("value" => $value, "source" => "http://apsc.gov.au/stateoftheservice/"); + $doc->statistics->employees->$timePeriod = Array("value" => $value, "source" => "http://apsc.gov.au/stateoftheservice/"); } } if ($changed) { --- /dev/null +++ b/admin/importAPSCEmployees2012.php @@ -1,1 +1,86 @@ +get_db('disclosr-agencies'); +$rows = $db->get_view("app", "byName")->rows; +$nametoid = Array(); +$sums = Array(); +$functions = Array(); +foreach ($rows as $row) { + $nametoid[trim($row->key)] = $row->value; +} + + +$request = Requests::get("http://www.apsc.gov.au/publications-and-media/parliamentary/state-of-the-service/new-sosr/appendix-2-aps-agencies"); +$doc = phpQuery::newDocumentHTML($request->body); +phpQuery::selectDocument($doc); +foreach (pq('tr')->elements as $tr) { + //echo $tr->nodeValue.PHP_EOL; + $agency = ""; + $employees = ""; + $function = ""; + $i = 0; + foreach ($tr->childNodes as $td) { + //echo $td->nodeValue." $i
"; + if ($i == 0) + $agency = $td->nodeValue; + if ($i == 2) { + $employees = trim(str_replace(",", "", $td->nodeValue)); + } + if ($i == 4) { + $function = $td->nodeValue; + } + $i++; + } + if ($agency != "" && $employees != "" && $function != "") { + $name = trim(str_replace('2','',$agency)); + //echo "$name

" . PHP_EOL; + if (isset($nametoid[$name])) { + $id = $nametoid[$name]; + //echo $id . "
" . PHP_EOL; + @$sums[$id]["2011-2012"] += $employees; + $functions[$id] = $function; + } else if ($agency != "Agency"){ + echo "
ERROR NAME '$agency' MISSING FROM ID LIST

" . PHP_EOL; + + die(); + } + } else { + echo "skipped $agency"; + } +} +//print_r($sums); +foreach ($sums as $id => $sum) { + echo $id . "
" . PHP_EOL; + $doc = $db->get($id); + echo $doc->name . "
" . PHP_EOL; + // print_r($doc); + $changed = false; + if (!isset($doc->statistics)) { + $changed = true; + $doc->statistics = new stdClass(); + } + if (!isset($doc->statistics->employees)) { + $changed = true; + $doc->statistics->employees = new stdClass(); + } + foreach ($sum as $timePeriod => $value) { + if (!isset($doc->statistics->employees->$timePeriod->value) + || $doc->statistics->employees->$timePeriod->value != $value) { + $changed = true; + $doc->statistics->employees->$timePeriod = Array("value" => $value, "source" => "http://apsc.gov.au/stateoftheservice/"); + $doc->employees = $value; + $doc->functionClassification = $functions[$id]; + } + } + + if ($changed) { + $db->save($doc); + } else { + echo "not changed" . "
" . PHP_EOL; + } +} +// employees: timeperiod, source = apsc state of service, value +?> + --- a/documents/disclogsList.php +++ b/documents/disclogsList.php @@ -19,47 +19,50 @@ if ($rows) { foreach ($rows as $row) { -if (!isset($row->value->status) || $row->value->status != "suspended") { - echo "". $row->value->name . ""; - if ($ENV == "DEV") - echo "
(" . $row->id . ")"; - echo "\n"; -$agencies++; + if ((!isset($row->value->status) || $row->value->status != "suspended") && isset($row->value->foiEmail)) { + echo ""; + if (isset($row->value->website)) echo ""; + echo "" . $row->value->name . ""; + if (isset($row->value->website)) echo ""; + if ($ENV == "DEV") + echo "
(" . $row->id . ")"; + echo "\n"; + $agencies++; - echo ""; - if (isset($row->value->FOIDocumentsURL)) { - $disclogs++; - echo '' - . $row->value->FOIDocumentsURL . ''; - if ($ENV == "DEV") - echo '
(' - . 'view local copy)'; - } else { - echo "✘"; - } - echo "\n"; - if (isset($row->value->FOIDocumentsURL)) { - if (file_exists("./scrapers/" . $row->id . '.py')) { - echo "✔"; - $green++; - } else if (file_exists("./scrapers/" . $row->id . '.txt')) { - echo "▬"; - $orange++; + echo ""; + if (isset($row->value->FOIDocumentsURL)) { + $disclogs++; + echo '' + . $row->value->FOIDocumentsURL . ''; + if ($ENV == "DEV") + echo '
(' + . 'view local copy)'; } else { echo "✘"; - $red++; } + echo "\n"; + if (isset($row->value->FOIDocumentsURL)) { + if (file_exists("./scrapers/" . $row->id . '.py')) { + echo "✔"; + $green++; + } else if (file_exists("./scrapers/" . $row->id . '.txt')) { + echo "▬"; + $orange++; + } else { + echo "✘"; + $red++; + } + } + echo "\n"; } - echo "\n"; } } -} } catch (SetteeRestClientException $e) { setteErrorHandler($e); } echo ""; -echo $agencies." agencies, ".round(($disclogs/$agencies)*100)."% with disclosure logs; " -.round(($green/$disclogs)*100)."% logs with scrapers ".round(($red/$disclogs)*100)."% logs without scrapers ".round(($orange/$disclogs)*100)."% logs Work-In-Progress scrapers "; +echo $agencies . " agencies, " . round(($disclogs / $agencies) * 100) . "% with disclosure logs; " + . round(($green / $disclogs) * 100) . "% logs with scrapers " . round(($red / $disclogs) * 100) . "% logs without scrapers " . round(($orange / $disclogs) * 100) . "% logs Work-In-Progress scrapers "; include_footer_documents(); ?> --- /dev/null +++ b/documents/scrapers/1d404c4934f74feacd00dcb434e7c10a.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + #def getTable(self,soup): + # return soup.find(id = "cphMain_C001_Col01").table + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/794ea270edc9aa4f70f2a84bbc5ecc7a.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "cphMain_C001_Col01").table + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/9f4815bfdcb918a036e4bb43a30f8d77.txt @@ -1,1 +1,1 @@ - +no disclog --- /dev/null +++ b/documents/scrapers/bf16d4ba0d306ee03e5a1d32aaba3da1.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(summary="This table shows every FOI request to date.") + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/e0614dc3a9e25d375370ffd82f7165ac.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + #def getTable(self,soup): + # return soup.find(id = "ctl00_PlaceHolderMain_intro2__ControlWrapper_CerRichHtmlField").table + def getColumnCount(self): + return 6 + def getColumns(self,columns): + (id, date, title, description,deldate, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() +