beginning datagov scraper
beginning datagov scraper


Former-commit-id: a8775a64a3cdda480e4433742ed7ea6ca6a437ef

<!doctype html> <!doctype html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<title>Page Not Found :(</title> <title>Page Not Found :(</title>
<style> <style>
::-moz-selection { background: #fe57a1; color: #fff; text-shadow: none; } ::-moz-selection {
::selection { background: #fe57a1; color: #fff; text-shadow: none; } background: #fe57a1;
html { padding: 30px 10px; font-size: 20px; line-height: 1.4; color: #737373; background: #f0f0f0; -webkit-text-size-adjust: 100%; -ms-text-size-adjust: 100%; } color: #fff;
html, input { font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; } text-shadow: none;
body { max-width: 500px; _width: 500px; padding: 30px 20px 50px; border: 1px solid #b3b3b3; border-radius: 4px; margin: 0 auto; box-shadow: 0 1px 10px #a7a7a7, inset 0 1px 0 #fff; background: #fcfcfc; } }
h1 { margin: 0 10px; font-size: 50px; text-align: center; }  
h1 span { color: #bbb; } ::selection {
h3 { margin: 1.5em 0 0.5em; } background: #fe57a1;
p { margin: 1em 0; } color: #fff;
ul { padding: 0 0 0 40px; margin: 1em 0; } text-shadow: none;
.container { max-width: 380px; _width: 380px; margin: 0 auto; } }
/* google search */  
#goog-fixurl ul { list-style: none; padding: 0; margin: 0; } html {
#goog-fixurl form { margin: 0; } padding: 30px 10px;
#goog-wm-qt, #goog-wm-sb { border: 1px solid #bbb; font-size: 16px; line-height: normal; vertical-align: top; color: #444; border-radius: 2px; } font-size: 20px;
#goog-wm-qt { width: 220px; height: 20px; padding: 5px; margin: 5px 10px 0 0; box-shadow: inset 0 1px 1px #ccc; } line-height: 1.4;
#goog-wm-sb { display: inline-block; height: 32px; padding: 0 10px; margin: 5px 0 0; white-space: nowrap; cursor: pointer; background-color: #f5f5f5; background-image: -webkit-linear-gradient(rgba(255,255,255,0), #f1f1f1); background-image: -moz-linear-gradient(rgba(255,255,255,0), #f1f1f1); background-image: -ms-linear-gradient(rgba(255,255,255,0), #f1f1f1); background-image: -o-linear-gradient(rgba(255,255,255,0), #f1f1f1); -webkit-appearance: none; -moz-appearance: none; appearance: none; *overflow: visible; *display: inline; *zoom: 1; } color: #737373;
#goog-wm-sb:hover, #goog-wm-sb:focus { border-color: #aaa; box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1); background-color: #f8f8f8; } background: #f0f0f0;
#goog-wm-qt:focus, #goog-wm-sb:focus { border-color: #105cb6; outline: 0; color: #222; } -webkit-text-size-adjust: 100%;
input::-moz-focus-inner { padding: 0; border: 0; } -ms-text-size-adjust: 100%;
</style> }
   
  html, input {
  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
  }
   
  body {
  max-width: 500px;
  _width: 500px;
  padding: 30px 20px 50px;
  border: 1px solid #b3b3b3;
  border-radius: 4px;
  margin: 0 auto;
  box-shadow: 0 1px 10px #a7a7a7, inset 0 1px 0 #fff;
  background: #fcfcfc;
  }
   
  h1 {
  margin: 0 10px;
  font-size: 50px;
  text-align: center;
  }
   
  h1 span {
  color: #bbb;
  }
   
  h3 {
  margin: 1.5em 0 0.5em;
  }
   
  p {
  margin: 1em 0;
  }
   
  ul {
  padding: 0 0 0 40px;
  margin: 1em 0;
  }
   
  .container {
  max-width: 380px;
  _width: 380px;
  margin: 0 auto;
  }
   
  /* google search */
  #goog-fixurl ul {
  list-style: none;
  padding: 0;
  margin: 0;
  }
   
  #goog-fixurl form {
  margin: 0;
  }
   
  #goog-wm-qt, #goog-wm-sb {
  border: 1px solid #bbb;
  font-size: 16px;
  line-height: normal;
  vertical-align: top;
  color: #444;
  border-radius: 2px;
  }
   
  #goog-wm-qt {
  width: 220px;
  height: 20px;
  padding: 5px;
  margin: 5px 10px 0 0;
  box-shadow: inset 0 1px 1px #ccc;
  }
   
  #goog-wm-sb {
  display: inline-block;
  height: 32px;
  padding: 0 10px;
  margin: 5px 0 0;
  white-space: nowrap;
  cursor: pointer;
  background-color: #f5f5f5;
  background-image: -webkit-linear-gradient(rgba(255, 255, 255, 0), #f1f1f1);
  background-image: -moz-linear-gradient(rgba(255, 255, 255, 0), #f1f1f1);
  background-image: -ms-linear-gradient(rgba(255, 255, 255, 0), #f1f1f1);
  background-image: -o-linear-gradient(rgba(255, 255, 255, 0), #f1f1f1);
  -webkit-appearance: none;
  -moz-appearance: none;
  appearance: none;
  *overflow: visible;
  *display: inline;
  *zoom: 1;
  }
   
  #goog-wm-sb:hover, #goog-wm-sb:focus {
  border-color: #aaa;
  box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
  background-color: #f8f8f8;
  }
   
  #goog-wm-qt:focus, #goog-wm-sb:focus {
  border-color: #105cb6;
  outline: 0;
  color: #222;
  }
   
  input::-moz-focus-inner {
  padding: 0;
  border: 0;
  }
  </style>
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<h1>Not found <span>:(</span></h1> <h1>Not found <span>:(</span></h1>
   
<p>Sorry, but the page you were trying to view does not exist.</p> <p>Sorry, but the page you were trying to view does not exist.</p>
   
<p>It looks like this was the result of either:</p> <p>It looks like this was the result of either:</p>
<ul> <ul>
<li>a mistyped address</li> <li>a mistyped address</li>
<li>an out-of-date link</li> <li>an out-of-date link</li>
</ul> </ul>
<script> <script>
var GOOG_FIXURL_LANG = (navigator.language || '').slice(0,2),GOOG_FIXURL_SITE = location.host; var GOOG_FIXURL_LANG = (navigator.language || '').slice(0, 2), GOOG_FIXURL_SITE = location.host;
</script> </script>
<script src="http://linkhelp.clients.google.com/tbproxy/lh/wm/fixurl.js"></script> <script src="http://linkhelp.clients.google.com/tbproxy/lh/wm/fixurl.js"></script>
</div> </div>
   
   
<?php <?php
include('template.inc.php'); include('template.inc.php');
include_once('../include/common.inc.php'); include_once('../include/common.inc.php');
$agenciesdb = $server->get_db('disclosr-agencies'); $agenciesdb = $server->get_db('disclosr-agencies');
   
$idtoname = Array(); $idtoname = Array();
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) { foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name); $idtoname[$row->id] = trim($row->value->name);
} }
$foidocsdb = $server->get_db('disclosr-foidocuments'); $foidocsdb = $server->get_db('disclosr-foidocuments');
   
include_header_documents((isset($_REQUEST['id']) ? $idtoname[$_REQUEST['id']] : 'Entries by Agency')); include_header_documents((isset($_REQUEST['id']) ? $idtoname[$_REQUEST['id']] : 'Entries by Agency'));
$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99'); $endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
?> ?>
<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div> <div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act
<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br> in one place!
  </div>
  <a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a>
  <br>
<?php <?php
try { try {
if ($_REQUEST['id']) { if ($_REQUEST['id']) {
$rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows; $rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows;
foreach ($rows as $row) { foreach ($rows as $row) {
//print_r($rows); //print_r($rows);
echo displayLogEntry($row, $idtoname); echo displayLogEntry($row, $idtoname);
if (!isset($startkey)) if (!isset($startkey))
$startkey = $row->key; $startkey = $row->key;
$endkey = $row->key; $endkey = $row->key;
} }
} else { } else {
$rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows; $rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows;
if ($rows) { if ($rows) {
foreach ($rows as $row) { foreach ($rows as $row) {
echo '<a href="agency.php?id=' . $row->key . '">' . $idtoname[$row->key] . " (" . $row->value . " records)</a> <br>\n"; echo '<a href="agency.php?id=' . $row->key . '">' . $idtoname[$row->key] . " (" . $row->value . " records)</a> <br>\n";
} }
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>"; echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
include_footer_documents(); include_footer_documents();
?> ?>
<?php <?php
include('template.inc.php'); include('template.inc.php');
include_header_documents("Charts"); include_header_documents("Charts");
include_once('../include/common.inc.php'); include_once('../include/common.inc.php');
$agenciesdb = $server->get_db('disclosr-agencies'); $agenciesdb = $server->get_db('disclosr-agencies');
   
$idtoname = Array(); $idtoname = Array();
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) { foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name); $idtoname[$row->id] = trim($row->value->name);
} }
$foidocsdb = $server->get_db('disclosr-foidocuments'); $foidocsdb = $server->get_db('disclosr-foidocuments');
   
?> ?>
<div class="foundation-header"> <div class="foundation-header">
<h1><a href="about.php">Charts</a></h1> <h1><a href="about.php">Charts</a></h1>
<h4 class="subheader">Lorem ipsum.</h4> <h4 class="subheader">Lorem ipsum.</h4>
</div> </div>
<div id="bydate" style="width:1000px;height:300px;"></div> <div id="bydate" style="width:1000px;height:300px;"></div>
<div id="byagency" style="width:1200px;height:300px;"></div> <div id="byagency" style="width:1200px;height:300px;"></div>
<script id="source"> <script id="source">
window.onload = function() { window.onload = function () {
$(document).ready(function() { $(document).ready(function () {
var var
d1 = [], d1 = [],
options1, options1,
o1; o1;
   
<?php <?php
try { try {
$rows = $foidocsdb->get_view("app", "byDateMonthYear?group=true",null, false,false,true)->rows; $rows = $foidocsdb->get_view("app", "byDateMonthYear?group=true",null, false,false,true)->rows;
   
   
$dataValues = Array(); $dataValues = Array();
foreach ($rows as $row) { foreach ($rows as $row) {
$dataValues[$row->key] = $row->value; $dataValues[$row->key] = $row->value;
} }
$i = 0; $i = 0;
ksort($dataValues); ksort($dataValues);
foreach ($dataValues as $key => $value) { foreach ($dataValues as $key => $value) {
$date = date_create_from_format('Y-m-d', $key); $date = date_create_from_format('Y-m-d', $key);
if (date_format($date, 'U') != "") { if (date_format($date, 'U') != "") {
echo " d1.push([".date_format($date, 'U')."000, $value]);" . PHP_EOL; echo " d1.push([".date_format($date, 'U')."000, $value]);" . PHP_EOL;
// echo " emplabels.push('$key');" . PHP_EOL; // echo " emplabels.push('$key');" . PHP_EOL;
$i++; $i++;
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
?> ?>
   
   
  options1 = {
options1 = { xaxis: {
xaxis : { mode: 'time',
mode : 'time', labelsAngle: 45
labelsAngle : 45 },
}, selection: {
selection : { mode: 'x'
mode : 'x' },
}, HtmlText: false,
HtmlText : false, title: 'Time'
title : 'Time' };
};  
   
// Draw graph with default options, overwriting with passed options  
function drawGraph (opts) {  
   
// Clone the options, so the 'options' variable always keeps intact. // Draw graph with default options, overwriting with passed options
o1 = Flotr._.extend(Flotr._.clone(options1), opts || {}); function drawGraph(opts) {
   
// Return a new graph. // Clone the options, so the 'options' variable always keeps intact.
return Flotr.draw( o1 = Flotr._.extend(Flotr._.clone(options1), opts || {});
document.getElementById("bydate"),  
[ d1 ],  
o1  
);  
}  
   
graph = drawGraph(); // Return a new graph.
  return Flotr.draw(
Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:select', function(area){ document.getElementById("bydate"),
// Draw selected area [ d1 ],
graph = drawGraph({ o1
xaxis : { min : area.x1, max : area.x2, mode : 'time', labelsAngle : 45 }, );
yaxis : { min : area.y1, max : area.y2 } }
});  
}); graph = drawGraph();
   
// When graph is clicked, draw the graph with default area. Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:select', function (area) {
Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:click', function () { graph = drawGraph(); }); // Draw selected area
  graph = drawGraph({
  xaxis: { min: area.x1, max: area.x2, mode: 'time', labelsAngle: 45 },
  yaxis: { min: area.y1, max: area.y2 }
  });
  });
   
  // When graph is clicked, draw the graph with default area.
  Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:click', function () {
  graph = drawGraph();
  });
   
}); });
}; };
   
var d2 = []; var d2 = [];
var agencylabels = []; var agencylabels = [];
function agencytrackformatter(obj) { function agencytrackformatter(obj) {
   
return agencylabels[Math.floor(obj.x)] +" = "+obj.y; return agencylabels[Math.floor(obj.x)] + " = " + obj.y;
   
} }
function agencytickformatter(val, axis) { function agencytickformatter(val, axis) {
if (agencylabels[Math.floor(val)]) { if (agencylabels[Math.