beginnings of document scrapers mark 2
Former-commit-id: fc862f3bada775436406e0ee8adcd4e7d04ac8f6
--- a/.gitmodules
+++ b/.gitmodules
@@ -25,4 +25,7 @@
[submodule "lib/querypath"]
path = lib/querypath
url = https://github.com/technosophos/querypath.git
+[submodule "lib/amon-php"]
+ path = lib/amon-php
+ url = https://github.com/martinrusev/amon-php.git
--- a/about.php
+++ b/about.php
@@ -10,8 +10,10 @@
Disclo.gs is a project to monitor Australian Federal Government agencies
compliance with their <a href="http://www.oaic.gov.au/publications/other_operational/foi_policy_frequently_asked_questions.html#_Toc291837571">"proactive disclosure requirements" to make a transparency league table as suggested by gov2 taskforce http://gov2.net.au/blog/2009/09/19/a-league-ladder-of-psi-openness/</a>.
<h2> Attributions </h2>
-National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm.
-
+National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm <br/>
+data.gov.au http://data.gov.au/dataset/directory-gov-au-full-data-export/ <br/>
+directory.gov.au <br/>
+australia.gov.au http://australia.gov.au/about/copyright <br/>
<h2> Open everything </h2>
All documents released CC-BY 3 AU
Open source git @
--- a/admin/naa-agift-tree.php
+++ b/admin/naa-agift-tree.php
@@ -31,9 +31,9 @@
<div id="tree-container"></div>
<div id="viz"></div>
-<script type="text/javascript" src="../javascripts/jquery-1.6.2.min.js"></script>
-<script type="text/javascript" src="../javascripts/d3.min.js"></script>
-<script type="text/javascript" src="../javascripts/d3.layout.min.js"></script>
+<script type="text/javascript" src="../js/jquery-1.6.2.min.js"></script>
+<script type="text/javascript" src="../js/d3.min.js"></script>
+<script type="text/javascript" src="../js/d3.layout.min.js"></script>
<script>
/**
--- a/alaveteli/exportAgencies.csv.php
+++ b/alaveteli/exportAgencies.csv.php
@@ -1,20 +1,6 @@
<?php
include_once("../include/common.inc.php");
-
-function shortName($name) {
- $name = trim($name);
- if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
- $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
- return str_replace($badWords,"",$name);
- }
-
- else {
- $out = Array();
- preg_match_all('/[A-Z]/', $name, $out);
- return implode("", $out[0]);
- }
-}
setlocale(LC_CTYPE, 'C');
@@ -63,38 +49,36 @@
$row = Array();
$row["#id"] = $agency->id;
$row["name"] = trim($agency->value->name);
- if (isset($agency->value->foiEmail)) {
- $row["request_email"] = $agency->value->foiEmail;
- } else {
- if ($agency->value->orgType == "FMA-DepartmentOfState") {
- $row["request_email"] = "foi@" . GetDomain($agency->value->website);
- } else {
- $row["request_email"] = $foiEmail[$agency->value->parentOrg];
+ $row["request_email"] = (isset($agency->value->foiEmail) ? $agency->value->foiEmail : "");
+ $row["short_name"] = (isset($agency->value->shortName) ? $agency->value->shortName : "");
+ $row["notes"] = (isset($agency->value->description) ? $agency->value->description : "");
+
+ $otherBodies = Array();
+ if (isset($agency->value->foiBodies)) {
+ $otherBodies = array_merge($otherBodies, $agency->value->foiBodies);
+ }
+ if (isset($agency->value->positions)) {
+ $positions = Array();
+ foreach ($agency->value->positions as $position) {
+ $positions[] = "Office of the ".$position;
}
+ $otherBodies = array_merge($otherBodies, $positions);
}
- if (isset($agency->value->shortName)) {
- $row["short_name"] = $agency->value->shortName;
- } else {
- $row["short_name"] = shortName($agency->value->name);
+ sort($otherBodies);
+ if (count($otherBodies) > 0) {
+ $row["notes"] .= "<br/> This department also responds to requests for information held by " . implode(", ", $otherBodies);
}
- $row["notes"] = "";
+
$row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
$row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
if ($agency->value->orgType == "FMA-DepartmentOfState") {
- $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
+ $row["tag_string"] = $tag[$agency->value->_id];
} else {
- $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
+ $row["tag_string"] = $tag[$agency->value->parentOrg];
}
-
+ $row["tag_string"] .= " " . $agency->value->orgType;
+ $row["tag_string"] .= " federal";
fputcsv($fp, array_values($row));
-
- if (isset($agency->value->foiBodies)) {
- foreach ($agency->value->foiBodies as $foiBody) {
- $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
- $row["short_name"] = shortName($foiBody);
- fputcsv($fp, array_values($row));
- }
- }
}
}
} catch (SetteeRestClientException $e) {
--- a/bubbletree.php
+++ b/bubbletree.php
@@ -5,13 +5,13 @@
<meta charset="UTF-8"/>
<title>Minimal BubbleTree Demo</title>
<script type="text/javascript" src="http://code.jquery.com/jquery-1.7.2.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/jquery.history.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/raphael.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/vis4.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/Tween.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/build/bubbletree.js"></script>
- <link rel="stylesheet" type="text/css" href="javascripts/bubbletree/build/bubbletree.css" />
- <script type="text/javascript" src="javascripts/bubbletree/styles/cofog.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/jquery.history.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/raphael.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/vis4.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/Tween.js"></script>
+ <script type="text/javascript" src="js/bubbletree/build/bubbletree.js"></script>
+ <link rel="stylesheet" type="text/css" href="js/bubbletree/build/bubbletree.css" />
+ <script type="text/javascript" src="js/bubbletree/styles/cofog.js"></script>
<script type="text/javascript">
--- /dev/null
+++ b/documents/google676a414ad086cefb.html
@@ -1,1 +1,2 @@
+google-site-verification: google676a414ad086cefb.html
--- /dev/null
+++ b/documents/index.php
@@ -1,1 +1,15 @@
+australian disclosure logs
+are you looking for more information about:
+contracts
+gov orgs
+lobbyists
+
+1/1/11 title (Dept dfggdfgdf)
+description:
+source link:
+documents:
+ #1 title link
+
+
+rss feed here
--- /dev/null
+++ b/documents/rss.xml.php
@@ -1,1 +1,30 @@
+<?php
+// Agency X updated Y, new files, diff of plain text/link text,
+// feed for just one agency or all
+// This is a minimum example of using the Universal Feed Generator Class
+include("lib/FeedWriter.php");
+//Creating an instance of FeedWriter class.
+$TestFeed = new FeedWriter(RSS2);
+//Setting the channel elements
+//Use wrapper functions for common channelelements
+$TestFeed->setTitle('Last Modified - All');
+$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php');
+$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
+//Retriving informations from database
+$rows = $db->get_view("app", "byLastModified")->rows;
+//print_r($rows);
+foreach ($rows as $row) {
+ //Create an empty FeedItem
+ $newItem = $TestFeed->createNewItem();
+ //Add elements to the feed item
+ $newItem->setTitle($row['name']);
+ $newItem->setLink($row['id']);
+ $newItem->setDate(date("c", $row['metadata']['lastModified']));
+ $newItem->setDescription($row['name']);
+ //Now add the feed item
+ $TestFeed->addItem($newItem);
+}
+//OK. Everything is done. Now genarate the feed.
+$TestFeed->genarateFeed();
+?>
--- /dev/null
+++ b/documents/runScrapers.php
--- /dev/null
+++ b/documents/scrapers/3cd40b1240e987cbcd3f0e67054ce259.py
@@ -1,1 +1,1 @@
-
+http://www.apvma.gov.au/rss/disclosure.rss
--- /dev/null
+++ b/documents/scrapers/8c9421f852c441910bf1d93a57b31d64.py
@@ -1,1 +1,1 @@
-
+http://www.daff.gov.au/about/foi/ips/disclosure_log
--- /dev/null
+++ b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
@@ -1,1 +1,2 @@
+http://foi.deewr.gov.au/disclosure-log/rss
--- /dev/null
+++ b/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
@@ -1,1 +1,1 @@
-
+www.finance.gov.au/foi/disclosure-log/foi-rss.xml
--- /dev/null
+++ b/documents/scrapers/rtk.py
@@ -1,1 +1,1 @@
-
+http://www.righttoknow.org.au/feed/search/%20(latest_status:successful%20OR%20latest_status:partially_successful)
--- /dev/null
+++ b/documents/search.php
@@ -1,1 +1,25 @@
+<?php
+include_once('include/common.inc.php');
+include_header('Search');
+?>
+<div class="foundation-header">
+ <h1><a href="search.php">Search</a></h1>
+</div>
+<form>
+ <input type="text" name="q" value="<?php if (isset($_REQUEST['q']))echo $_REQUEST['q'];?>"/>
+ <input type="submit"/>
+</form>
+<?php
+if (isset($_REQUEST['q'])) {
+ $request = Requests::get($serverAddr."disclosr-documents/_fti/_design/lucene/by_all?include_docs=true&q=".$_REQUEST['q']);
+ $results = json_decode($request->body);
+ $db = $server->get_db('disclosr-documents');
+ foreach ($results->rows as $result) {
+ //print_r($result);
+ //$row = $db->get($result->id);
+ echo $result->doc->_id." ".$result->doc->url."<br>".PHP_EOL;
+ }
+}
+include_footer();
+?>
--- a/getAgency.php
+++ b/getAgency.php
@@ -9,7 +9,11 @@
if (strpos($key, "_") === 0 || $key== "metadata") return;
echo "<tr>";
- echo "<td>" . $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small></td><td>";
+ echo "<td>";
+ if (isset($schemas['agency']["properties"][$key])) {
+ echo $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description']."</small>";
+ }
+ echo "</td><td>";
if (is_array($value)) {
echo "<ol>";
foreach ($value as $subkey => $subvalue) {
--- a/graph.php
+++ b/graph.php
@@ -86,9 +86,9 @@
if ($format == "html") {
?>
<div id="sigma-example" width="960" style="min-height:800px;background-color: #333;"></div>
- <script src="javascripts/sigma.min.js"></script>
- <script src="javascripts/sigma/plugins/sigma.parseGexf.js"></script>
- <script src="javascripts/sigma/plugins/sigma.forceatlas2.js"></script>
+ <script src="js/sigma.min.js"></script>
+ <script src="js/sigma/plugins/sigma.parseGexf.js"></script>
+ <script src="js/sigma/plugins/sigma.forceatlas2.js"></script>
<script type="text/javascript">function init() {
// Instanciate sigma.js and customize rendering :
var sigInst = sigma.init(document.getElementById('sigma-example')).drawingProperties({
--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -14,6 +14,11 @@
require_once $basePath.'lib/Requests/library/Requests.php';
Requests::register_autoloader();
+require $basePath."lib/amon-php/amon.php";
+Amon::config(array('address'=> 'http://127.0.0.1:2464',
+ 'protocol' => 'http',
+ 'secret_key' => "I2LJ6dOMmlnXgVAkTPFXd5M3ejkga8Gd2FbBt6iqZdw"));
+Amon::setup_exception_handler();
# Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735
function object_to_array(stdClass $Class) {
--- a/include/couchdb.inc.php
+++ b/include/couchdb.inc.php
@@ -168,6 +168,7 @@
$server = new SetteeServer($serverAddr);
function setteErrorHandler($e) {
+ Amon::log($e->getMessage() . " " . print_r($_SERVER,true), array('error'));
echo $e->getMessage() . "<br>" . PHP_EOL;
}
--- a/include/template.inc.php
+++ b/include/template.inc.php
@@ -66,11 +66,11 @@
<!-- Included JS Files -->
- <script src="<?php echo $basePath; ?>javascripts/foundation.js"></script>
- <script src="<?php echo $basePath; ?>javascripts/app.js"></script>
+ <script src="<?php echo $basePath; ?>js/foundation.js"></script>
+ <script src="<?php echo $basePath; ?>js/app.js"></script>
<script src="http://code.jquery.com/jquery-1.7.1.min.js"></script>
- <script type="text/javascript" src="javascripts/flotr2/flotr2.js"></script>
+ <script type="text/javascript" src="<?php echo $basePath ?>js/flotr2/flotr2.js"></script>
<?php
if (strpos($_SERVER['SERVER_NAME'], ".gs")) {
?>
--- /dev/null
+++ b/js/bubbletree
--- /dev/null
+++ b/js/flotr2
--- /dev/null
+++ b/js/sigma
--- /dev/null
+++ b/lib/amon-php
--- a/rss.xml.php
+++ /dev/null
@@ -1,30 +1,1 @@
-<?php
-// Agency X updated Y, new files, diff of plain text/link text,
-// feed for just one agency or all
-// This is a minimum example of using the Universal Feed Generator Class
-include("lib/FeedWriter.php");
-//Creating an instance of FeedWriter class.
-$TestFeed = new FeedWriter(RSS2);
-//Setting the channel elements
-//Use wrapper functions for common channelelements
-$TestFeed->setTitle('Last Modified - All');
-$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php');
-$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
-//Retriving informations from database
-$rows = $db->get_view("app", "byLastModified")->rows;
-//print_r($rows);
-foreach ($rows as $row) {
- //Create an empty FeedItem
- $newItem = $TestFeed->createNewItem();
- //Add elements to the feed item
- $newItem->setTitle($row['name']);
- $newItem->setLink($row['id']);
- $newItem->setDate(date("c", $row['metadata']['lastModified']));
- $newItem->setDescription($row['name']);
- //Now add the feed item
- $TestFeed->addItem($newItem);
-}
-//OK. Everything is done. Now genarate the feed.
-$TestFeed->genarateFeed();
-?>
--- a/search.php
+++ /dev/null
@@ -1,25 +1,1 @@
-<?php
-include_once('include/common.inc.php');
-include_header('Search');
-?>
-<div class="foundation-header">
- <h1><a href="search.php">Search</a></h1>
-</div>
-<form>
- <input type="text" name="q" value="<?php if (isset($_REQUEST['q']))echo $_REQUEST['q'];?>"/>
- <input type="submit"/>
-</form>
-<?php
-if (isset($_REQUEST['q'])) {
- $request = Requests::get($serverAddr."disclosr-documents/_fti/_design/lucene/by_all?include_docs=true&q=".$_REQUEST['q']);
- $results = json_decode($request->body);
- $db = $server->get_db('disclosr-documents');
- foreach ($results->rows as $result) {
- //print_r($result);
- //$row = $db->get($result->id);
- echo $result->doc->_id." ".$result->doc->url."<br>".PHP_EOL;
- }
-}
-include_footer();
-?>