Scrape required and chart of complied features views
Former-commit-id: 1f70b43713b7686e9f7a8a0f6a5aced655d53221
--- /dev/null
+++ b/admin/refreshDesignDoc.php
@@ -1,1 +1,7 @@
+<?php
+require_once '../include/common.inc.php';
+$db = $server->get_db('disclosr-agencies');
+createAgencyDesignDoc();
+?>
+
--- a/alaveteli/exportAgencies.csv.php
+++ b/alaveteli/exportAgencies.csv.php
@@ -1,6 +1,21 @@
<?php
include_once("../include/common.inc.php");
+
+function shortName($name) {
+ $name = trim($name);
+ if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
+ $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
+ return str_replace($badWords,"",$name);
+ }
+
+ else {
+ $out = Array();
+ preg_match_all('/[A-Z]/', $name, $out);
+ return implode("", $out[0]);
+ }
+}
+
setlocale(LC_CTYPE, 'C');
$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
@@ -12,15 +27,28 @@
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows);
foreach ($rows as $row) {
- $tag[$row->value] = phrase_to_tag(dept_to_portfolio($row->key));
+ $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
+ die();
+}
+
+$foiEmail = Array();
+try {
+ $rows = $db->get_view("app", "foiEmails", null, true)->rows;
+ //print_r($rows);
+ foreach ($rows as $row) {
+ $foiEmail[$row->key] = $row->value;
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ die();
}
$fp = fopen('php://output', 'w');
if ($fp && $db) {
- header('Content-Type: text/csv');
+ header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache');
header('Expires: 0');
@@ -29,39 +57,43 @@
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows);
foreach ($agencies as $agency) {
- // print_r($agency);
- $row = Array();
- $row["#id"] = $agency->value->_id;
- $row["name"] = $agency->value->name;
- if (isset($agency->value->foi_email)) {
- $row["request_email"] = $agency->value->foi_email;
- } else {
- $row["request_email"] = "foi@".GetDomain($agency->value->website);
- //
- }
- if (isset($agency->value->short_name)) {
- $row["short_name"] = $agency->value->short_name;
- } else {
- $out = Array();
- preg_match_all('/[A-Z]/', $agency->value->name, $out);
- $row["short_name"] = implode("",$out[0]);
- }
- $row["notes"] = "";
- $row["publication_scheme"] = $agency->value->infoPublicationSchemeURL;
- $row["home_page"] = $agency->value->website;
- if ($agency->value->orgType == "FMA-DepartmentOfState") {
- $row["tag_string"] = $tag[$agency->value->_id];
- } else {
- $row["tag_string"] = $tag[$agency->value->parentOrg];;
- }
-
- fputcsv($fp, array_values($row));
-
- if (isset($agency->value->foiBodies)) {
- foreach ($agency->value->foiBodies as $foiBody) {
- $row['name'] = $foiBody;
- $row['short_name'] = "";
- fputcsv($fp, array_values($row));
+ // print_r($agency);
+
+ if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
+ $row = Array();
+ $row["#id"] = $agency->id;
+ $row["name"] = trim($agency->value->name);
+ if (isset($agency->value->foiEmail)) {
+ $row["request_email"] = $agency->value->foiEmail;
+ } else {
+ if ($agency->value->orgType == "FMA-DepartmentOfState") {
+ $row["request_email"] = "foi@" . GetDomain($agency->value->website);
+ } else {
+ $row["request_email"] = $foiEmail[$agency->value->parentOrg];
+ }
+ }
+ if (isset($agency->value->shortName)) {
+ $row["short_name"] = $agency->value->shortName;
+ } else {
+ $row["short_name"] = shortName($agency->value->name);
+ }
+ $row["notes"] = "";
+ $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
+ $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
+ if ($agency->value->orgType == "FMA-DepartmentOfState") {
+ $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
+ } else {
+ $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
+ }
+
+ fputcsv($fp, array_values($row));
+
+ if (isset($agency->value->foiBodies)) {
+ foreach ($agency->value->foiBodies as $foiBody) {
+ $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
+ $row["short_name"] = shortName($foiBody);
+ fputcsv($fp, array_values($row));
+ }
}
}
}
--- a/alaveteli/exportCategories.rb.php
+++ b/alaveteli/exportCategories.rb.php
@@ -1,19 +1,20 @@
<?php
+
include_once("../include/common.inc.php");
setlocale(LC_CTYPE, 'C');
- header('Content-Type: text/csv');
- header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
- header('Pragma: no-cache');
- header('Expires: 0');
-echo 'PublicBodyCategories.add(:en, ['.PHP_EOL;
-echo ' "Portfolios",'.PHP_EOL;
+header('Content-Type: text/csv');
+header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
+header('Pragma: no-cache');
+header('Expires: 0');
+echo 'PublicBodyCategories.add(:en, [' . PHP_EOL;
+echo ' "Portfolios",' . PHP_EOL;
$db = $server->get_db('disclosr-agencies');
try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows);
foreach ($rows as $row) {
- echo ' [ "'.phrase_to_tag(dept_to_portfolio($row->key)).'","'. dept_to_portfolio($row->key).'","'.dept_to_portfolio($row->key).'" ],'.PHP_EOL;
+ echo ' [ "' . phrase_to_tag(dept_to_portfolio($row->key)) . '","' . dept_to_portfolio($row->key) . '","part of the ' . dept_to_portfolio($row->key) . ' portfolio" ],' . PHP_EOL;
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
--- a/getAgency.php
+++ b/getAgency.php
@@ -4,17 +4,36 @@
include_header();
function displayValue($key, $value, $mode) {
- global $db;
+ global $db, $schemas;
if ($mode == "view") {
+
+ echo "<tr>";
+
+ echo "<td>" . $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small></td><td>";
if (is_array($value)) {
- echo "<tr><td>$key</td><td><ol>";
+ echo "<ol>";
foreach ($value as $subkey => $subvalue) {
- echo "<li>$subvalue</li>";
+ if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
+ echo '<li itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
+ } else {
+ echo "<li>";
+ }
+ echo "$subvalue</li>";
}
echo "</ol></td></tr>";
} else {
- echo "<tr><td>$key</td><td>$value</td></tr>";
+ if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
+ echo '<span itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
+ } else {
+ echo "<span>";
+ }
+ if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
+ echo "<a href='$value'>view</a></span>";
+ } else {
+ echo "$value</span>";
+ }
}
+ echo "</td></tr>";
}
if ($mode == "edit") {
if (is_array($value)) {
@@ -32,22 +51,22 @@
if (strpos($key, "_") === 0) {
echo"<input type='hidden' id='$key' name='$key' value='$value'/>";
} else if ($key == "parentOrg") {
- echo "<label for='$key'>$key</label><select id='$key' name='$key'>";
- $rows = $db->get_view("app", "byDeptStateName")->rows;
- //print_r($rows);
- foreach ($rows as $row) {
- echo "<option value='{$row->value}'".(($row->value == $value) ? "SELECTED":"")." >".str_replace("Department of ","",$row->key)."</option>";
- }
- echo" </select>";
+ echo "<label for='$key'>$key</label><select id='$key' name='$key'><option value=''> Select... </option>";
+ $rows = $db->get_view("app", "byDeptStateName")->rows;
+ //print_r($rows);
+ foreach ($rows as $row) {
+ echo "<option value='{$row->value}'" . (($row->value == $value) ? "SELECTED" : "") . " >" . str_replace("Department of ", "", $row->key) . "</option>";
+ }
+ echo" </select>";
} else if (strpos($key, "has") === 0) {
- echo "<label for='$key'><input type='checkbox' id='$key' name='$key' ".(($value=='true')?"checked='$value'":"")."> $key</label>";
+ echo "<label for='$key'><input type='checkbox' id='$key' name='$key' " . (($value == 'on' || $value == 'true') ? "checked='$value'" : "") . "> $key</label>";
} else {
echo "<label>$key</label><input class='input-text' type='text' id='$key' name='$key' value='$value'/>";
- if ((strpos($key,"URL") > 0 || $key == 'website')&& $value != "") {
- echo "<a href='$value'>view</a>";
+ if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
+ echo "<a href='$value'>view</a>";
}
if ($key == 'abn') {
- echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>";
+ echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>";
}
}
}
@@ -62,16 +81,14 @@
if (!isset($row[$defaultField])) {
if ($schemas['agency']['properties'][$defaultField]['type'] == "string") {
if (strpos($defaultField, "has") === 0) {
- $row[$defaultField] = "false";
-
+ $row[$defaultField] = "false";
} else {
- $row[$defaultField] = "";
+ $row[$defaultField] = "";
}
-
}
- if ($schemas['agency']['properties'][$defaultField]['type'] == "array") {
-
- $row[$defaultField] = Array("");
+ if ($schemas['agency']['properties'][$defaultField]['type'] == "array") {
+
+ $row[$defaultField] = Array("");
}
}
}
@@ -89,20 +106,33 @@
//print_r($row);
if (sizeof($_POST) > 0) {
//print_r($_POST);
+ foreach ($_POST as $postkey => $postvalue) {
+ if ($postvalue == "") {
+ unset($_POST[$postkey]);
+ }
+ if (is_array($postvalue) && count($postvalue) == 1 && $postvalue[0] == "") {
+ unset($_POST[$postkey]);
+ }
+ }
if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) {
echo "Edited version was latest version, continue saving";
$newdoc = $_POST;
$newdoc['metadata']['lastModified'] = time();
$row = $db->save($newdoc);
} else {
- echo "ALERT doc revised by someone else while editing.";
+ echo "ALERT doc revised by someone else while editing. Document not saved.";
}
}
- $mode = "edit";
- $row = addDefaultFields(object_to_array($row));
+ $mode = "view";
+ if ($mode == "edit") {
+ $row = addDefaultFields(object_to_array($row));
+ } else {
+ $row = object_to_array($row);
+ }
+
if ($mode == "view") {
- echo '<table width="100%">';
+ echo '<div itemscope itemtype ="http://schema.org/GovernmentOrganisation"><table width="100%">';
echo '<tr> <td colspan="2"><h3>' . $row['name'] . "</h3></td></tr>";
echo "<tr><th>Field Name</th><th>Field Value</th></tr>";
}
@@ -127,40 +157,40 @@
};
</script>
<form id="editform" class="nice" method="post">
- <?php
+ <?php
+ }
+ foreach ($row as $key => $value) {
+ echo displayValue($key, $value, $mode);
+ }
+ if ($mode == "view") {
+ echo "</table></div>";
+ }
+ if ($mode == "edit") {
+ echo '<input id="submitbutton" type="submit"/></form>';
+ }
+} else {
+
+ try {
+ /* $rows = $db->get_view("app", "showNamesABNs")->rows;
+ //print_r($rows);
+ foreach ($rows as $row) {
+ // print_r($row);
+ echo '<li><a href="getAgency.php?id=' . $row->key . '">' .
+ (isset($row->value->name) && $row->value->name != "" ? $row->value->name : "NO NAME " . $row->value->abn)
+ . '</a></li>';
+ } */
+ $rows = $db->get_view("app", "byName")->rows;
+ //print_r($rows);
+ foreach ($rows as $row) {
+ // print_r($row);
+ echo '<li itemscope itemtype="http://schema.org/GovernmentOrganization"><a href="getAgency.php?id=' . $row->value . '" itemprop="url"><span itemprop="name">' .
+ $row->key
+ . '</span></a></li>';
}
- foreach ($row as $key => $value) {
- echo displayValue($key, $value, $mode);
- }
- if ($mode == "view") {
- echo "</table>";
- }
- if ($mode == "edit") {
- echo '<input id="submitbutton" type="submit"/></form>';
- }
- } else {
-
- try {
- /*$rows = $db->get_view("app", "showNamesABNs")->rows;
- //print_r($rows);
- foreach ($rows as $row) {
- // print_r($row);
- echo '<li><a href="getAgency.php?id=' . $row->key . '">' .
- (isset($row->value->name) && $row->value->name != "" ? $row->value->name : "NO NAME " . $row->value->abn)
- . '</a></li>';
- }*/
- $rows = $db->get_view("app", "byName")->rows;
- //print_r($rows);
- foreach ($rows as $row) {
- // print_r($row);
- echo '<li><a href="getAgency.php?id=' . $row->value . '">' .
- $row->key
- . '</a></li>';
- }
- } catch (SetteeRestClientException $e) {
- setteErrorHandler($e);
- }
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
}
- include_footer();
- ?>
+}
+include_footer();
+?>
--- a/graph.php
+++ b/graph.php
@@ -1,21 +1,51 @@
<?php
include_once('include/common.inc.php');
//include_header();
-?>
-<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script>
-<script src="lib/springy/springy.js"></script>
-<script src="lib/springy/springyui.js"></script>
-<script>
- var graph = new Graph();
- var nodes = [];
-<?php
+$format = "html";
+if (isset($_REQUEST['format'])) {
+ $format = $_REQUEST['format'];
+}
+
+function add_node($id, $label) {
+ global $format;
+ if ($format == "html") {
+ echo "nodes[\"$id\"] = graph.newNode({label: \"$label\"});" . PHP_EOL;
+ }
+ if ($format == "dot" && $label != "") {
+ echo "$id [label=\"$label\"];". PHP_EOL;
+ }
+}
+
+function add_edge($from, $to, $color) {
+ global $format;
+ if ($format == "html") {
+ echo "graph.newEdge(nodes[\"$from\"], nodes['$to'], {color: '$color'});" . PHP_EOL;
+ }
+ if ($format == "dot") {
+ echo "$from -> $to ".($color != ""? "[color=$color]":"").";". PHP_EOL;
+ }
+}
+
+if ($format == "html") {
+ ?>
+ <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"></script>
+ <script src="lib/springy/springy.js"></script>
+ <script src="lib/springy/springyui.js"></script>
+ <script>
+ var graph = new Graph();
+ var nodes = [];
+ <?php
+}
+if ($format == "dot") {
+ echo 'digraph g {'. PHP_EOL;
+}
$db = $server->get_db('disclosr-agencies');
-echo "nodes[\"fedg\"] = graph.newNode({label: \"Federal Government - Commonwealth of Australia\"});" . PHP_EOL;
+ add_node("fedg","Federal Government - Commonwealth of Australia");
try {
$rows = $db->get_view("app", "byCanonicalName", null, true)->rows;
- //print_r($rows);
+//print_r($rows);
foreach ($rows as $row) {
- echo "nodes[\"{$row->value->_id}\"] = graph.newNode({label: \"{$row->key}\"});" . PHP_EOL;
+ add_node($row->id, $row->key);
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
@@ -23,9 +53,9 @@
try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
- //print_r($rows);
+//print_r($rows);
foreach ($rows as $row) {
- echo "graph.newEdge(nodes[\"fedg\"], nodes['{$row->value}'], {color: '#00A0B0'});" . PHP_EOL;
+ add_edge("fedg", $row->value, 'yellow');
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
@@ -33,25 +63,30 @@
try {
$rows = $db->get_view("app", "parentOrgs", null, true)->rows;
- // print_r($rows);
+// print_r($rows);
foreach ($rows as $row) {
- echo "graph.newEdge(nodes[\"{$row->key}\"], nodes['{$row->value}'], {color: '#FFA0B0'});" . PHP_EOL;
+ add_edge($row->key, $row->value, 'blue');
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
}
-?>
- window.onload = function() {
- $(document).ready(function() {
- var springy = $('#springydemo').springy({
- graph: graph
+if ($format == "html") {
+ ?>
+ window.onload = function() {
+ $(document).ready(function() {
+ var springy = $('#springydemo').springy({
+ graph: graph
+ });
});
- });
- };
-</script>
+ };
+ </script>
-<canvas id="springydemo" width="1260" height="680" />
-<?php
+ <canvas id="springydemo" width="1260" height="680" />
+ <?php
+}
+if ($format == "dot") {
+ echo "}";
+}
//include_footer();
?>
--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -47,7 +47,7 @@
return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName)));
}
function phrase_to_tag ($phrase) {
- return str_replace(" ","_",str_replace(",","",strtolower($phrase)));
+ return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase))));
}
function GetDomain($url)
{
@@ -62,7 +62,3 @@
}
}
-?>
-
-
-
--- a/include/couchdb.inc.php
+++ b/include/couchdb.inc.php
@@ -1,8 +1,8 @@
<?php
-include $basePath."schemas/schemas.inc.php";
+include $basePath . "schemas/schemas.inc.php";
-require ($basePath.'couchdb/settee/src/settee.php');
+require ($basePath . 'couchdb/settee/src/settee.php');
function createAgencyDesignDoc() {
global $db;
@@ -11,17 +11,17 @@
$obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
- $obj->views->byCanonicalName->map = "function(doc) {
+ $obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
- emit(doc.name, doc._id);
+ emit(doc.name, doc);
}
};";
- $obj->views->byDeptStateName->map = "function(doc) {
+ $obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id);
}
};";
- $obj->views->parentOrgs->map = "function(doc) {
+ $obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) {
emit(doc._id, doc.parentOrg);
}
@@ -34,10 +34,25 @@
}
}
};";
+
+ $obj->views->foiEmails->map = "function(doc) {
+ emit(doc._id, doc.foiEmail);
+};";
+
$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
- $obj->views->getScrapeRequired->map = "function(doc) { emit(doc.abn, doc); };";
+ $obj->views->getScrapeRequired->map = "function(doc) {
+
+var lastScrape = Date.parse(doc.metadata.lastScraped);
+
+var today = new Date();
+
+if (!lastScrape || lastScrape.getTime() + 1000 != today.getTime()) {
+ emit(doc._id, doc);
+}
+
+};";
$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
$obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) {
@@ -45,6 +60,26 @@
}
}";
// http://stackoverflow.com/questions/646628/javascript-startswith
+ $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
+ String.prototype.startsWith = function (str) {
+ return !this.indexOf(str);
+ }
+}
+if(!String.prototype.endsWith){
+ String.prototype.endsWith = function(suffix) {
+ return this.indexOf(suffix, this.length - suffix.length) !== -1;
+ };
+}
+function(doc) {
+if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
+for(var propName in doc) {
+ if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") || propName.endsWith("URL"))) {
+ emit(propName, 1);
+ }
+}
+ emit("total", 1);
+ }
+}';
$obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {
return !this.indexOf(str);
@@ -67,19 +102,17 @@
return $db->save($obj, true);
}
+if (php_uname('n') == "vanille") {
-if( php_uname('n') == "vanille") {
+ $server = new SetteeServer('http://192.168.178.21:5984');
+} else
+if (php_uname('n') == "KYUUBEY") {
-$server = new SetteeServer('http://192.168.178.21:5984');
-} else
- if( php_uname('n') == "KYUUBEY") {
-
-$server = new SetteeServer('http://192.168.1.148:5984');
+ $server = new SetteeServer('http://192.168.1.148:5984');
} else {
$server = new SetteeServer('http://127.0.0.1:5984');
}
+
function setteErrorHandler($e) {
echo $e->getMessage() . "<br>" . PHP_EOL;
}
-?>
-
--- a/include/template.inc.php
+++ b/include/template.inc.php
@@ -19,11 +19,11 @@
<title>Disclosr</title>
<!-- Included CSS Files -->
- <link rel="stylesheet" href="<?php echo $basePath?>stylesheets/foundation.css">
- <link rel="stylesheet" href="<?php echo $basePath?>stylesheets/app.css">
+ <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/foundation.css">
+ <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/app.css">
<!--[if lt IE 9]>
- <link rel="stylesheet" href="<?php echo $basePath?>stylesheets/ie.css">
+ <link rel="stylesheet" href="<?php echo $basePath ?>stylesheets/ie.css">
<![endif]-->
@@ -44,7 +44,7 @@
<div class="eight columns hide-on-phones">
<strong class="right">
<a href="getAgency.php">Agencies</a>
- <a href="about.php">About/FAQ</a>
+ <a href="about.php">About/FAQ</a>
</strong>
</div>
</div>
@@ -55,7 +55,10 @@
<div class="container">
<?php }
- function include_footer() { ?>
+ function include_footer() {
+ global $basePath;
+ ?>
+
</div>
<!-- container -->
@@ -63,14 +66,11 @@
<!-- Included JS Files -->
- <script src="<?php echo $basePath?>javascripts/foundation.js"></script>
- <script src="<?php echo $basePath?>javascripts/app.js"></script>
+ <script src="<?php echo $basePath; ?>javascripts/foundation.js"></script>
+ <script src="<?php echo $basePath; ?>javascripts/app.js"></script>
<script src="http://code.jquery.com/jquery-1.7.1.min.js"></script>
</body>
</html>
-<?php }
-
-?>
-
+<?php }
--- a/schemas/agency.json.php
+++ b/schemas/agency.json.php
@@ -4,38 +4,44 @@
"description" => "Representation of government agency and online transparency measures",
"type" => "object",
"properties" => Array(
- "name" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "short_name" => Array("type" => "string", "required" => false, "x-title"=> "Agency Short Name", "description" => "Agency Name shortened, usually to an acronym"),
- "otherNames" => Array("type" => "array", "required" => true, "x-title" => "Agency Past/Other Names", "description" => "Agency Names",
+ "name" => Array("type" => "string", "required" => true, "x-itemprop" => "name", "x-title" => "Name", "description" => "Name, most recent and broadest"),
+ "shortName" => Array("type" => "string", "required" => false, "x-title" => "Short Name", "description" => "Name shortened, usually to an acronym"),
+ "foiEmail" => Array("type" => "string", "required" => false, "x-title" => "FOI Contact Email", "description" => "FOI contact email if not foi@"),
+ "sameAs" => Array("type" => "array", "required" => false, "x-itemprop"=>"http://www.w3.org/2002/07/owl#sameAs","x-title" => "Same As", "description" => "Same as other URLs/URIs for this entity",
"items" => Array("type" => "string")),
- "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies", "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous",
+ "otherNames" => Array("type" => "array", "required" => true, "x-title" => "Past/Other Names", "description" => "Other names for organisation",
"items" => Array("type" => "string")),
- "orgType"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "parentOrg"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "website"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "abn" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "contractListURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "grantsReportingURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "annualReportURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "consultanciesURL" => Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "legalExpenditureURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "recordsListURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "FOIDocumentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "infoPublicationSchemeURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "appointmentsURL"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "hasRSS"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "hasMailingList"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "hasTwitter"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "hasFacebook"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
- "hasYouTube"=> Array("type" => "string", "required" => true, "x-title"=> "Agency Name", "description" => "Agency Name, most recent and broadest"),
-
+ "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies","x-itemprop"=>"members", "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous",
+ "items" => Array("type" => "string")),
+ "orgType" => Array("type" => "string", "required" => true, "x-title" => "Organisation Type", "description" => "Org type based on legal formation via FMA/CAC legislation etc."),
+ "parentOrg" => Array("type" => "string", "required" => true, "x-title" => "Parent Organisation", "description" => "Parent organisation, usually a department of state"),
+ "website" => Array("type" => "string", "required" => true, "x-title" => "Website", "x-itemprop" => "url", "description" => "Website URL"),
+ "abn" => Array("type" => "string", "required" => true, "x-title" => "Australian Business Number", "description" => "ABN from business register"),
+ "contractListURL" => Array("type" => "string", "required" => true, "x-title" => "Contract Listing", "description" => "Departmental and agency contracts, <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a>"),
+ "grantsReportingURL" => Array("type" => "string", "required" => true, "x-title" => "Grants Awarded",
+ "description" => "Departmental and agency grants <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a> and <a href='http://www.finance.gov.au/publications/fmg-series/23-commonwealth-grant-guidelines.html'>Commonwealth grants guidelines</a> "),
+ "annualReportURL" => Array("type" => "string", "required" => true, "x-title" => "Annual Report(s)", "description" => ""),
+ "consultanciesURL" => Array("type" => "string", "required" => true, "x-title" => "Consultants Hired", "description" => ""),
+ "legalExpenditureURL" => Array("type" => "string", "required" => true, "x-title" => "Legal Services Expenditure", "description" => "Legal Services Expenditure mandated by Legal Services Directions 2005"),
+ "recordsListURL" => Array("type" => "string", "required" => true, "x-title" => "Files/Records Held", "description" => "Indexed lists of departmental and agency files, <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a>"),
+ "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => ""),
+ "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""),
+ "appointmentsURL" => Array("type" => "string", "required" => true, "x-title" => "Agency Appointments/Boards", "description" => "Departmental and agency appointments and vacancies , <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a>"),
+ "advertisingURL" => Array("type" => "string", "required" => true, "x-title" => "Approved Advertising Campaigns", "description" => " Agency advertising and public information projects, <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a> "),
+ "hasRSS" => Array("type" => "string", "required" => true, "x-title" => "Has RSS", "description" => ""),
+ "hasMailingList" => Array("type" => "string", "required" => true, "x-title" => "Has Mailing List", "description" => ""),
+ "hasTwitter" => Array("type" => "string", "required" => true, "x-title" => "Has Twitter", "description" => ""),
+ "hasFacebook" => Array("type" => "string", "required" => true, "x-title" => "Has Facebook", "description" => ""),
+ "hasYouTube" => Array("type" => "string", "required" => true, "x-title" => "Has YouTube", "description" => ""),
+ "hasFlickr" => Array("type" => "string", "required" => true, "x-title" => "Has Flickr", "description" => ""),
+ "hasCCBY" => Array("type" => "string", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution"),
),
- /*"org":{"type":"object",
- "properties":{
- "organizationName":{"type":"string"},
- "organizationUnit":{"type":"string"}},
- }
- }*/
+ /* "org":{"type":"object",
+ "properties":{
+ "organizationName":{"type":"string"},
+ "organizationUnit":{"type":"string"}},
+ }
+ } */
);
?>
--- /dev/null
+++ b/scrape.py
@@ -1,1 +1,76 @@
+#http://packages.python.org/CouchDB/client.html
+import couchdb
+import urllib2
+from BeautifulSoup import BeautifulSoup
+import re
+#http://diveintopython.org/http_web_services/etags.html
+class NotModifiedHandler(urllib2.BaseHandler):
+ def http_error_304(self, req, fp, code, message, headers):
+ addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
+ addinfourl.code = code
+ return addinfourl
+
+def scrapeAndStore(URL, depth, agency):
+ URL = "http://www.google.com"
+ req = urllib2.Request(URL)
+ etag = 'y'
+ last_modified = 'y'
+ #if there is a previous version sotred in couchdb, load caching helper tags
+ if etag:
+ req.add_header("If-None-Match", etag)
+ if last_modified:
+ req.add_header("If-Modified-Since", last_modified)
+
+ opener = urllib2.build_opener(NotModifiedHandler())
+ url_handle = opener.open(req)
+ headers = url_handle.info() # the addinfourls have the .info() too
+ etag = headers.getheader("ETag")
+ last_modified = headers.getheader("Last-Modified")
+ web_server = headers.getheader("Server")
+ file_size = headers.getheader("Content-Length")
+ mime_type = headers.getheader("Content-Type")
+
+ if hasattr(url_handle, 'code'):
+ if url_handle.code == 304:
+ print "the web page has not been modified"
+ else:
+ #do scraping
+ html = url_handle.read()
+ # http://www.crummy.com/software/BeautifulSoup/documentation.html
+ soup = BeautifulSoup(html)
+ links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-"))
+ for link in links:
+ print link['href']
+ #for each unique link
+ #if html mimetype
+ # go down X levels,
+ # diff with last stored attachment, store in document
+ #if not
+ # remember to save parentURL and title (link text that lead to document)
+
+ #store as attachment epoch-filename
+ else:
+ print "error %s in downloading %s", url_handle.code, URL
+ #record/alert error to error database
+
+
+
+
+
+
+
+
+
+
+
+couch = couchdb.Server('http://192.168.1.148:5984/')
+
+# select database
+agencydb = couch['disclosr-agencies']
+
+for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view?
+ agency = agencydb.get(row.id)
+ print agency['name']
+scrapeAndStore("A",1,1)
+
--- a/unimplemented/exportAgencies.csv.php
+++ /dev/null
@@ -1,65 +1,1 @@
-<?php
-include_once("./lib/common.inc.php");
-setlocale(LC_CTYPE, 'C');
-// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
-
-$unspsc = Array();
-$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';');
-$unspscresult->execute();
-foreach ($unspscresult->fetchAll() as $row) {
- $unspsc[$row['UNSPSC']] = $row['Title'];
-}
-
-$query = $conn->prepare('
-SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN",
-EXTRACT(EPOCH FROM "publishDate") as "publishDate",
-EXTRACT(EPOCH FROM "contractStart") as "contractStart",
-EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",
-value,description,category,
-"supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
-(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
-FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName"
-where "childCN" is null'
- , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
-$query->execute();
-$errors = $conn->errorInfo();
-if ($errors[2] != "") {
- die("Export terminated, db error" . print_r($errors, true));
-}
-
-$num_fields = $query->columnCount();
-$headers = Array();
-for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header
- $meta = $query->getColumnMeta($i);
- $headers[] = $meta['name'];
-}
-$fp = fopen('php://output', 'w');
-if ($fp && $query) {
- header('Content-Type: text/csv');
- header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
- header('Pragma: no-cache');
- header('Expires: 0');
- fputcsv($fp, $headers);
- while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
- foreach ($row as $key => &$colvalue) {
-
- $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($colvalue));
- if ($headers[$key] == "publishDate" || $headers[$key] == "contractStart"
- || $headers[$key] == "contractEnd") {
- $colvalue = date("Y-m-d", $colvalue);
- }
- /* if ($headers[$key] == "CNID") {
- $colvalue = str_replace("A","", $colvalue);
-}*/
- if ($headers[$key] == "cat1" || $headers[$key] == "cat2"
- || $headers[$key] == "cat3") {
- $colvalue = $unspsc[$colvalue];
- }
- }
- fputcsv($fp, array_values($row));
- }
- die;
-}
-?>
-
--- a/unimplemented/scrape.py
+++ /dev/null
@@ -1,64 +1,1 @@
-#http://packages.python.org/CouchDB/client.html
-import couchdb
-import urllib2
-from BeautifulSoup import BeautifulSoup
-import re
-couch = couchdb.Server() # Assuming localhost:5984
-# If your CouchDB server is running elsewhere, set it up like this:
-# couch = couchdb.Server('http://example.com:5984/')
-
-# select database
-agencydb = couch['disclosr-agencies']
-
-for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view?
- agency = agencydb.get(row.id)
- print agency['agencyName']
-
-#http://diveintopython.org/http_web_services/etags.html
-class NotModifiedHandler(urllib2.BaseHandler):
- def http_error_304(self, req, fp, code, message, headers):
- addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
- addinfourl.code = code
- return addinfourl
-
-def scrapeAndStore(URL, depth, agency):
- URL = "http://www.hole.fi/jajvirta/weblog/"
- req = urllib2.Request(URL)
-
- #if there is a previous version sotred in couchdb, load caching helper tags
- if etag:
- req.add_header("If-None-Match", etag)
- if last_modified:
- req.add_header("If-Modified-Since", last_modified)
-
- opener = urllib2.build_opener(NotModifiedHandler())
- url_handle = opener.open(req)
- headers = url_handle.info() # the addinfourls have the .info() too
- etag = headers.getheader("ETag")
- last_modified = headers.getheader("Last-Modified")
- web_server = headers.getheader("Server")
- file_size = headers.getheader("Content-Length")
- mime_type = headers.getheader("Content-Type")
-
- if hasattr(url_handle, 'code') and url_handle.code == 304:
- print "the web page has not been modified"
- else:
- print "error %s in downloading %s", url_handle.code, URL
- #record/alert error to error database
-
- #do scraping
- html = ?
- # http://www.crummy.com/software/BeautifulSoup/documentation.html
- soup = BeautifulSoup(html)
-links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-"))
-for link in links:
- print link['href']
- #for each unique link
- #if html mimetype
- # go down X levels,
- # diff with last stored attachment, store in document
- #if not
- # remember to save parentURL and title (link text that lead to document)
-
- #store as attachment epoch-filename