gitphp 0.2.9.1 :: disclosr.git/commitdiff

file:a/couchdb/settee/src/classes/SetteeDatabase.class.php -> file:b/couchdb/settee/src/classes/SetteeDatabase.class.php

<?php	<?php

/**	/**
* Databaase class.	* Databaase class.
*/	*/
class SetteeDatabase {	class SetteeDatabase {

/**	/**
* Base URL of the CouchDB REST API	* Base URL of the CouchDB REST API
*/	*/
private $conn_url;	private $conn_url;

/**	/**
* HTTP REST Client instance	* HTTP REST Client instance
*/	*/
protected $rest_client;	protected $rest_client;

/**	/**
* Name of the database	* Name of the database
*/	*/
private $dbname;	private $dbname;

/**	/**
* Default constructor	* Default constructor
*/	*/
function __construct($conn_url, $dbname) {	function __construct($conn_url, $dbname) {
$this->conn_url = $conn_url;	$this->conn_url = $conn_url;
$this->dbname = $dbname;	$this->dbname = $dbname;
$this->rest_client = SetteeRestClient::get_instance($this->conn_url);	$this->rest_client = SetteeRestClient::get_instance($this->conn_url);
}	}


/**	/**
* Get UUID from CouchDB	* Get UUID from CouchDB
*	*
* @return	* @return
* CouchDB-generated UUID string	* CouchDB-generated UUID string
*	*
*/	*/
function gen_uuid() {	function gen_uuid() {
$ret = $this->rest_client->http_get('_uuids');	$ret = $this->rest_client->http_get('_uuids');
return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking	return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking
}	}

/**	/**
* Create or update a document database	* Create or update a document database
*	*
* @param $document	* @param $document
* PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.	* PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.
*	*
* <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).	* <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).
* If "_id" is missing, CouchDB will be used to generate a UUID.	* If "_id" is missing, CouchDB will be used to generate a UUID.
*	*
* <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.	* <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.
* You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be	* You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be
* one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but	* one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but
* not provide "_id" since that is an invalid input.	* not provide "_id" since that is an invalid input.
*	*
* @param $allowRevAutoDetection	* @param $allowRevAutoDetection
* Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision	* Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision
* for a document and use it. This option is "false" by default because it involves an extra http HEAD request and	* for a document and use it. This option is "false" by default because it involves an extra http HEAD request and
* therefore can make save() operation slightly slower if such auto-detection is not required.	* therefore can make save() operation slightly slower if such auto-detection is not required.
*	*
* @return	* @return
* document object with the database id (uuid) and revision attached;	* document object with the database id (uuid) and revision attached;
*	*
* @throws SetteeCreateDatabaseException	* @throws SetteeCreateDatabaseException
*/	*/
function save($document, $allowRevAutoDetection = false) {	function save($document, $allowRevAutoDetection = false) {
if (is_string($document)) {	if (is_string($document)) {
$document = json_decode($document);	$document = json_decode($document);
}	}

// Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)	// Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)
if(is_array($document)) {	if(is_array($document)) {
$document = (object) $document;	$document = (object) $document;
}	}

if (empty($document->_id) && empty($document->_rev)) {	if (empty($document->_id) && empty($document->_rev)) {
$id = $this->gen_uuid();	$id = $this->gen_uuid();
}	}
elseif (empty($document->_id) && !empty($document->_rev)) {	elseif (empty($document->_id) && !empty($document->_rev)) {
throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");	throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");
}	}
else {	else {
$id = $document->_id;	$id = $document->_id;

if ($allowRevAutoDetection) {	if ($allowRevAutoDetection) {
try {	try {
$rev = $this->get_rev($id);	$rev = $this->get_rev($id);
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
// auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error	// auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error
}	}
if (!empty($rev)) {	if (!empty($rev)) {
$document->_rev = $rev;	$document->_rev = $rev;
}	}
}	}
}	}

$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);	$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
$document_json = json_encode($document, JSON_NUMERIC_CHECK);	$document_json = json_encode($document, JSON_NUMERIC_CHECK);

$ret = $this->rest_client->http_put($full_uri, $document_json);	$ret = $this->rest_client->http_put($full_uri, $document_json);

$document->_id = $ret['decoded']->id;	$document->_id = $ret['decoded']->id;
$document->_rev = $ret['decoded']->rev;	$document->_rev = $ret['decoded']->rev;

return $document;	return $document;
}	}

/**	/**
* @param $doc	* @param $doc
* @param $name	* @param $name
* @param $content	* @param $content
* Content of the attachment in a string-buffer format. This function will automatically base64-encode content for	* Content of the attachment in a string-buffer format. This function will automatically base64-encode content for
* you, so you don't have to do it.	* you, so you don't have to do it.
* @param $mime_type	* @param $mime_type
* Optional. Will be auto-detected if not provided	* Optional. Will be auto-detected if not provided
* @return void	* @return void
*/	*/
public function add_attachment($doc, $name, $content, $mime_type = null) {	public function add_attachment($doc, $name, $content, $mime_type = null) {
if (empty($doc->_attachments) \|\| !is_object($doc->_attachments)) {	if (empty($doc->_attachments) \|\| !is_object($doc->_attachments)) {
$doc->_attachments = new stdClass();	$doc->_attachments = new stdClass();
}	}

if (empty($mime_type)) {	if (empty($mime_type)) {
$mime_type = $this->rest_client->content_mime_type($content);	$mime_type = $this->rest_client->content_mime_type($content);
}	}

$doc->_attachments->$name = new stdClass();	$doc->_attachments->$name = new stdClass();
$doc->_attachments->$name->content_type = $mime_type;	$doc->_attachments->$name->content_type = $mime_type;
$doc->_attachments->$name->data = base64_encode($content);	$doc->_attachments->$name->data = base64_encode($content);
}	}

/**	/**
* @param $doc	* @param $doc
* @param $name	* @param $name
* @param $file	* @param $file
* Full path to a file (e.g. as returned by PHP's realpath function).	* Full path to a file (e.g. as returned by PHP's realpath function).
* @param $mime_type	* @param $mime_type
* Optional. Will be auto-detected if not provided	* Optional. Will be auto-detected if not provided
* @return void	* @return void
*/	*/
public function add_attachment_file($doc, $name, $file, $mime_type = null) {	public function add_attachment_file($doc, $name, $file, $mime_type = null) {
$content = file_get_contents($file);	$content = file_get_contents($file);
$this->add_attachment($doc, $name, $content, $mime_type);	$this->add_attachment($doc, $name, $content, $mime_type);
}	}

/**	/**
*	*
* Retrieve a document from CouchDB	* Retrieve a document from CouchDB
*	*
* @throws SetteeWrongInputException	* @throws SetteeWrongInputException
*	*
* @param $id	* @param $id
* Unique ID (usually: UUID) of the document to be retrieved.	* Unique ID (usually: UUID) of the document to be retrieved.
* @return	* @return
* database document in PHP object format.	* database document in PHP object format.
*/	*/
function get($id) {	function get($id) {
if (empty($id)) {	if (empty($id)) {
throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");	throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
}	}

$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);	$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri);	$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri);
$ret = $this->rest_client->http_get($full_uri);	$ret = $this->rest_client->http_get($full_uri);
return $ret['decoded'];	return $ret['decoded'];
}	}

/**	/**
*	*
* Get the latest revision of a document with document id: $id in CouchDB.	* Get the latest revision of a document with document id: $id in CouchDB.
*	*
* @throws SetteeWrongInputException	* @throws SetteeWrongInputException
*	*
* @param $id	* @param $id
* Unique ID (usually: UUID) of the document to be retrieved.	* Unique ID (usually: UUID) of the document to be retrieved.
* @return	* @return
* database document in PHP object format.	* database document in PHP object format.
*/	*/
function get_rev($id) {	function get_rev($id) {
if (empty($id)) {	if (empty($id)) {
throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");	throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");
}	}

$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);	$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
$headers = $this->rest_client->http_head($full_uri);	$headers = $this->rest_client->http_head($full_uri);
if (empty($headers['Etag'])) {	if (empty($headers['Etag'])) {
throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");	throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");
}	}
$etag = str_replace('"', '', $headers['Etag']);	$etag = str_replace('"', '', $headers['Etag']);
return $etag;	return $etag;
}	}

/**	/**
* Delete a document	* Delete a document
*	*
* @param $document	* @param $document
* a PHP object or JSON representation of the document that has _id and _rev fields.	* a PHP object or JSON representation of the document that has _id and _rev fields.
*	*
* @return void	* @return void
*/	*/
function delete($document) {	function delete($document) {
if (!is_object($document)) {	if (!is_object($document)) {
$document = json_decode($document);	$document = json_decode($document);
}	}

$full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;	$full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;
$this->rest_client->http_delete($full_uri);	$this->rest_client->http_delete($full_uri);
}	}


/----------------- View-related functions --------------/	/----------------- View-related functions --------------/

/**	/**
* Create a new view or update an existing one.	* Create a new view or update an existing one.
*	*
* @param $design_doc	* @param $design_doc
* @param $view_name	* @param $view_name
* @param $map_src	* @param $map_src
* Source code of the map function in Javascript	* Source code of the map function in Javascript
* @param $reduce_src	* @param $reduce_src
* Source code of the reduce function in Javascript (optional)	* Source code of the reduce function in Javascript (optional)
* @return void	* @return void
*/	*/
function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {	function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {
$obj = new stdClass();	$obj = new stdClass();
$obj->_id = "_design/" . urlencode($design_doc);	$obj->_id = "_design/" . urlencode($design_doc);
$view_name = urlencode($view_name);	$view_name = urlencode($view_name);
$obj->views->$view_name->map = $map_src;	$obj->views->$view_name->map = $map_src;
if (!empty($reduce_src)) {	if (!empty($reduce_src)) {
$obj->views->$view_name->reduce = $reduce_src;	$obj->views->$view_name->reduce = $reduce_src;
}	}

// allow safe updates (even if slightly slower due to extra: rev-detection check).	// allow safe updates (even if slightly slower due to extra: rev-detection check).
return $this->save($obj, true);	return $this->save($obj, true);
}	}

/**	/**
* Create a new view or update an existing one.	* Create a new view or update an existing one.
*	*
* @param $design_doc	* @param $design_doc
* @param $view_name	* @param $view_name
* @param $key	* @param $key
* key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes	* key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes
* that first element is startkey, second: endkey.	* that first element is startkey, second: endkey.
* @param $descending	* @param $descending
* return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change	* return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change
* order you also need to swap startkey and endkey values!	* order you also need to swap startkey and endkey values!
*	*
* @return void	* @return void
*/	*/
function get_view($design_doc, $view_name, $key = null, $descending = false, $limit = false) {	function get_view($design_doc, $view_name, $key = null, $descending = false, $limit = false, $reduce=false) {
$id = "_design/" . urlencode($design_doc);	$id = "_design/" . urlencode($design_doc);
$view_name = urlencode($view_name);	$view_name = urlencode($view_name);
$id .= "/_view/$view_name";	$id .= "/_view/$view_name";

$data = array();	$data = array();
if (!empty($key)) {	if (!empty($key)) {
if (is_string($key)) {	if (is_string($key)) {
$data = "key=" . '"' . $key . '"';	$data = "key=" . '"' . $key . '"';
}	}
elseif (is_array($key)) {	elseif (is_array($key)) {
list($startkey, $endkey) = $key;	list($startkey, $endkey) = $key;
$data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';	$data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';
}	}

if ($descending) {	if ($descending) {
$data .= "&descending=true";	$data .= "&descending=true";
}	}
	if ($reduce) {
	$data .= "&reduce=true";
	} else {
	$data .= "&reduce=false";
	}
if ($limit) {	if ($limit) {
$data .= "&limit=".$limit;	$data .= "&limit=".$limit;
}	}
}	}



if (empty($id)) {	if (empty($id)) {
throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");	throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
}	}

$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);	$full_uri = $this->dbname . "/" . $this->safe_urlencode($id);

$full_uri = str_replace("%253Fgroup%253D","?group=",$full_uri);	$full_uri = str_replace("%253Fgroup%253D","?group=",$full_uri);
$full_uri = str_replace("%253Flimit%253D","?limit=",$full_uri);	$full_uri = str_replace("%253Flimit%253D","?limit=",$full_uri);
$ret = $this->rest_client->http_get($full_uri, $data);	$ret = $this->rest_client->http_get($full_uri, $data);
	//$ret['decoded'] = str_replace("?k","&k",$ret['decoded']);
return $ret['decoded'];	return $ret['decoded'];

}	}

/**	/**
* @param $id	* @param $id
* @return	* @return
* return a properly url-encoded id.	* return a properly url-encoded id.
*/	*/
private function safe_urlencode($id) {	private function safe_urlencode($id) {
//-- System views like _design can have "/" in their URLs.	//-- System views like _design can have "/" in their URLs.
$id = rawurlencode($id);	$id = rawurlencode($id);
if (substr($id, 0, 1) == '_') {	if (substr($id, 0, 1) == '_') {
$id = str_replace('%2F', '/', $id);	$id = str_replace('%2F', '/', $id);
}	}
return $id;	return $id;
}	}

/** Getter for a database name */	/** Getter for a database name */
function get_name() {	function get_name() {
return $this->dbname;	return $this->dbname;
}	}

}	}

file:a/documents/about.php -> file:b/documents/about.php

<?php	<?php

include('template.inc.php');	include('template.inc.php');
include_header_documents("");	include_header_documents("About");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');
?>	?>
<h1>About</h1>	<h1>About</h1>
<?php	<?php
include_footer_documents();	include_footer_documents();
?>	?>

file:b/documents/agency.php (new)

	<?php
	include('template.inc.php');
	include_once('../include/common.inc.php');
	$agenciesdb = $server->get_db('disclosr-agencies');

	$idtoname = Array();
	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
	$idtoname[$row->id] = trim($row->value->name);
	}
	$foidocsdb = $server->get_db('disclosr-foidocuments');

	include_header_documents((isset($_REQUEST['id']) ? $idtoname[$_REQUEST['id']] : 'Entries by Agency'));
	$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
	?>
	<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
	<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
	<?php
	try {
	if ($_REQUEST['id']) {
	$rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows;
	foreach ($rows as $row) {
	//print_r($rows);
	echo displayLogEntry($row, $idtoname);
	if (!isset($startkey))
	$startkey = $row->key;
	$endkey = $row->key;
	}
	} else {
	$rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows;
	if ($rows) {
	foreach ($rows as $row) {
	echo '<a href="agency.php?id=' . $row->key . '">' . $idtoname[$row->key] . " (" . $row->value . " records)</a> <br>\n";
	}
	}
	}
	} catch (SetteeRestClientException $e) {
	setteErrorHandler($e);
	}
	echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
	include_footer_documents();
	?>

file:a/documents/charts.php -> file:b/documents/charts.php

<?php	<?php
include('template.inc.php');	include('template.inc.php');
include_header_documents("");	include_header_documents("Charts");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');
$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');

$idtoname = Array();	$idtoname = Array();
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name);	$idtoname[$row->id] = trim($row->value->name);
}	}
$foidocsdb = $server->get_db('disclosr-foidocuments');	$foidocsdb = $server->get_db('disclosr-foidocuments');

?>	?>
<div class="foundation-header">	<div class="foundation-header">
<h1><a href="about.php">Charts</a></h1>	<h1><a href="about.php">Charts</a></h1>
<h4 class="subheader">Lorem ipsum.</h4>	<h4 class="subheader">Lorem ipsum.</h4>
</div>	</div>
<div id="employees" style="width:1000px;height:900px;"></div>	<div id="bydate" style="width:1000px;height:300px;"></div>
	<div id="byagency" style="width:1200px;height:300px;"></div>
<script id="source">	<script id="source">
window.onload = function() {	window.onload = function() {
$(document).ready(function() {	$(document).ready(function() {
var	var
d1 = [],	d1 = [],
start = new Date("2009/01/01 01:00").getTime(),	options1,
options,	o1;
graph,
i, x, o;

<?php	<?php
try {	try {
$rows = $foidocsdb->get_view("app", "byDate?group=true", null, true)->rows;	$rows = $foidocsdb->get_view("app", "byDateMonthYear?group=true",null, false,false,true)->rows;


$dataValues = Array();	$dataValues = Array();
foreach ($rows as $row) {	foreach ($rows as $row) {
$dataValues[$row->value] = $row->key;	$dataValues[$row->key] = $row->value;
}	}
$i = 0;	$i = 0;
ksort($dataValues);	ksort($dataValues);
foreach ($dataValues as $value => $key) {	foreach ($dataValues as $key => $value) {
$date = date_create_from_format('Y-m-d', $key);	$date = date_create_from_format('Y-m-d', $key);
if (date_format($date, 'U') != "") {	if (date_format($date, 'U') != "") {
echo " d1.push([".date_format($date, 'U')."000, $value]);" . PHP_EOL;	echo " d1.push([".date_format($date, 'U')."000, $value]);" . PHP_EOL;
// echo " emplabels.push('$key');" . PHP_EOL;	// echo " emplabels.push('$key');" . PHP_EOL;
$i++;	$i++;
}	}
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
?>	?>



options = {	options1 = {
xaxis : {	xaxis : {
mode : 'time',	mode : 'time',
labelsAngle : 45	labelsAngle : 45
},	},
selection : {	selection : {
mode : 'x'	mode : 'x'
},	},
HtmlText : false,	HtmlText : false,
title : 'Time'	title : 'Time'
};	};

// Draw graph with default options, overwriting with passed options	// Draw graph with default options, overwriting with passed options
function drawGraph (opts) {	function drawGraph (opts) {

// Clone the options, so the 'options' variable always keeps intact.	// Clone the options, so the 'options' variable always keeps intact.
o = Flotr._.extend(Flotr._.clone(options), opts \|\| {});	o1 = Flotr._.extend(Flotr._.clone(options1), opts \|\| {});

// Return a new graph.	// Return a new graph.
return Flotr.draw(	return Flotr.draw(
document.getElementById("employees"),	document.getElementById("bydate"),
[ d1 ],	[ d1 ],
o	o1
);	);
}	}

graph = drawGraph();	graph = drawGraph();

Flotr.EventAdapter.observe(container, 'flotr:select', function(area){	Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:select', function(area){
// Draw selected area	// Draw selected area
graph = drawGraph({	graph = drawGraph({
xaxis : { min : area.x1, max : area.x2, mode : 'time', labelsAngle : 45 },	xaxis : { min : area.x1, max : area.x2, mode : 'time', labelsAngle : 45 },
yaxis : { min : area.y1, max : area.y2 }	yaxis : { min : area.y1, max : area.y2 }
});	});
});	});

// When graph is clicked, draw the graph with default area.	// When graph is clicked, draw the graph with default area.
Flotr.EventAdapter.observe(container, 'flotr:click', function () { graph = drawGraph(); });	Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:click', function () { graph = drawGraph(); });

});	});
};	};

	var d2 = [];
	var agencylabels = [];
	function agencytrackformatter(obj) {

	return agencylabels[Math.floor(obj.x)] +" = "+obj.y;

	}
	function agencytickformatter(val, axis) {
	if (agencylabels[Math.floor(val)]) {
	return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(agencylabels[Math.floor(val)])+"</b>";

	} else {
	return "";
	}
	}
	<?php
	try {
	$rows = $foidocsdb->get_view("app", "byAgencyID?group=true",null, false,false,true)->rows;


	$dataValues = Array();
	$i = 0;
	foreach ($rows as $row) {
	echo " d2.push([".$i.", $row->value]);" . PHP_EOL;
	echo " agencylabels.push(['".str_replace("'","",$idtoname[$row->key])."']);" . PHP_EOL;

	$i++;
	}
	} catch (SetteeRestClientException $e) {
	setteErrorHandler($e);
	}
	?>
	// Draw the graph
	Flotr.draw(
	document.getElementById("byagency"),
	[d2],
	{
	bars : {
	show : true,
	horizontal : false,
	shadowSize : 0,
	barWidth : 0.5
	},
	mouse : {
	track : true,
	relative : true,
	trackFormatter: agencytrackformatter
	},
	yaxis : {
	min : 0,
	autoscaleMargin : 1
	},
	xaxis: {
	minorTickFreq: 1,
	noTicks: agencylabels.length,
	showMinorLabels: true,
	tickFormatter: agencytickformatter
	},
	legend: {
	show: false
	}
	}
	);
</script>	</script>

<?php	<?php
include_footer_documents();	include_footer_documents();
?>	?>

file:b/documents/date.php (new)

	<?php

	include('template.inc.php');
	include_header_documents("Entries by Date");
	include_once('../include/common.inc.php');
	$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
	?>
	<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
	<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
	<?php
	/*$agenciesdb = $server->get_db('disclosr-agencies');

	$idtoname = Array();
	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
	$idtoname[$row->id] = trim($row->value->name);
	}
	$foidocsdb = $server->get_db('disclosr-foidocuments');
	try {
	$rows = $foidocsdb->get_view("app", "byDate", Array($endkey, '0000-00-00'), true, 20)->rows;
	if ($rows) {
	foreach ($rows as $key => $row) {
	echo displayLogEntry($row, $idtoname);
	if (!isset($startkey)) $startkey = $row->key;
	$endkey = $row->key;
	}
	}
	} catch (SetteeRestClientException $e) {
	setteErrorHandler($e);
	}
	echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
	*/
	include_footer_documents();
	?>

file:a/documents/disclogsList.php -> file:b/documents/disclogsList.php

<?php	<?php

include('template.inc.php');	include('template.inc.php');
include_header_documents("");	include_header_documents("List of Disclosure Logs");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');

echo "<table>	echo "<table>
<tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>";	<tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>";
$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');
$docsdb = $server->get_db('disclosr-documents');	$docsdb = $server->get_db('disclosr-documents');
$agencies = 0;	$agencies = 0;
$disclogs = 0;	$disclogs = 0;
$red = 0;	$red = 0;
$green = 0;	$green = 0;
$yellow = 0;	$yellow = 0;
$orange = 0;	$orange = 0;
try {	try {
$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;	$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;


if ($rows) {	if ($rows) {
foreach ($rows as $row) {	foreach ($rows as $row) {
if ((!isset($row->value->status) \|\| $row->value->status != "suspended") && isset($row->value->foiEmail)) {	if ((!isset($row->value->status) \|\| $row->value->status != "suspended") && isset($row->value->foiEmail)) {
echo "<tr><td>";	echo "<tr><td>";
if (isset($row->value->website)) echo "<a href='" . $row->value->website . "'>";	if (isset($row->value->website)) echo "<a href='" . $row->value->website . "'>";
echo "<b>" . $row->value->name . "</b>";	echo "<b>" . $row->value->name . "</b>";
if (isset($row->value->website)) echo "</a>";	if (isset($row->value->website)) echo "</a>";
if ($ENV == "DEV")	if ($ENV == "DEV")
echo "<br>(" . $row->id . ")";	echo "<br>(" . $row->id . ")";
echo "</td>\n";	echo "</td>\n";
$agencies++;	$agencies++;

echo "<td>";	echo "<td>";
if (isset($row->value->FOIDocumentsURL)) {	if (isset($row->value->FOIDocumentsURL)) {
$disclogs++;	$disclogs++;
echo '<a href="' . $row->value->FOIDocumentsURL . '">'	echo '<a href="' . $row->value->FOIDocumentsURL . '">'
. $row->value->FOIDocumentsURL . '</a>';	. $row->value->FOIDocumentsURL . '</a>';
if ($ENV == "DEV")	if ($ENV == "DEV")
echo '<br><small>(<a href="viewDocument.php?hash=' . md5($row->value->FOIDocumentsURL) . '">'	echo '<br><small>(<a href="viewDocument.php?hash=' . md5($row->value->FOIDocumentsURL) . '">'
. 'view local copy</a>)</small>';	. 'view local copy</a>)</small>';
} else {	} else {
echo "<font color='red'><abbr title='No'>✘</abbr></font>";	echo "<font color='red'><abbr title='No'>✘</abbr></font>";
}	}
echo "</td>\n<td>";	echo "</td>\n<td>";
if (isset($row->value->FOIDocumentsURL)) {	if (isset($row->value->FOIDocumentsURL)) {
if (file_exists("./scrapers/" . $row->id . '.py')) {	if (file_exists("./scrapers/" . $row->id . '.py')) {
echo "<font color='green'><abbr title='Yes'>✔</abbr></font>";	echo "<font color='green'><abbr title='Yes'>✔</abbr></font>";
$green++;	$green++;
} else if (file_exists("./scrapers/" . $row->id . '.txt')) {	} else if (file_exists("./scrapers/" . $row->id . '.txt')) {
if (trim(file_get_contents("./scrapers/" . $row->id . '.txt')) == "no disclog") {	if (trim(file_get_contents("./scrapers/" . $row->id . '.txt')) == "no disclog") {
echo "<font color='yellow'><abbr title='No log table exists at URL to scrape'><b>◎</b></abbr></font>";	echo "<font color='yellow'><abbr title='No log table exists at URL to scrape'><b>◎</b></abbr></font>";
$yellow++;	$yellow++;
} else {	} else {
echo file_get_contents("./scrapers/" . $row->id . '.txt');	echo file_get_contents("./scrapers/" . $row->id . '.txt');
echo "<font color='orange'><abbr title='Work in progress'><b>▬</b></abbr></font>";	echo "<font color='orange'><abbr title='Work in progress'><b>▬</b></abbr></font>";
$orange++;	$orange++;
}	}
} else {	} else {
echo "<font color='red'><abbr title='No'>✘</abbr></font>";	echo "<font color='red'><abbr title='No'>✘</abbr></font>";
$red++;	$red++;
}	}
}	}
echo "</td></tr>\n";	echo "</td></tr>\n";
}	}
}	}
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
echo "</table>";	echo "</table>";
echo $agencies . " agencies, " . round(($disclogs / $agencies) * 100) . "% with disclosure logs; "	echo $agencies . " agencies, " . round(($disclogs / $agencies) * 100) . "% with disclosure logs; "
. round(($green / $disclogs) * 100) . "% logs with scrapers " . round(($red / $disclogs) * 100) . "% logs without scrapers " . round(($orange / $disclogs) * 100) . "% logs Work-In-Progress scrapers ";	. round(($green / $disclogs) * 100) . "% logs with scrapers " . round(($red / $disclogs) * 100) . "% logs without scrapers " . round(($orange / $disclogs) * 100) . "% logs Work-In-Progress scrapers ";

include_footer_documents();	include_footer_documents();
?>	?>

file:a/documents/genericScrapers.py -> file:b/documents/genericScrapers.py

import sys	import sys
import os	import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import scrape	import scrape
from bs4 import BeautifulSoup	from bs4 import BeautifulSoup
from time import mktime	from time import mktime
import feedparser	import feedparser
import abc	import abc
import unicodedata	import unicodedata
import re	import re
import dateutil	import dateutil
from dateutil.parser import *	from dateutil.parser import *
from datetime import *	from datetime import *
import codecs	import codecs

from StringIO import StringIO	from StringIO import StringIO

from pdfminer.pdfparser import PDFDocument, PDFParser	from pdfminer.pdfparser import PDFDocument, PDFParser
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter, process_pdf	from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter, process_pdf
from pdfminer.pdfdevice import PDFDevice, TagExtractor	from pdfminer.pdfdevice import PDFDevice, TagExtractor
from pdfminer.converter import TextConverter	from pdfminer.converter import TextConverter
from pdfminer.cmapdb import CMapDB	from pdfminer.cmapdb import CMapDB
from pdfminer.layout import LAParams	from pdfminer.layout import LAParams


class GenericDisclogScraper(object):	class GenericDisclogScraper(object):
__metaclass__ = abc.ABCMeta	__metaclass__ = abc.ABCMeta
agencyID = None	agencyID = None
disclogURL = None	disclogURL = None

def remove_control_chars(self, input):	def remove_control_chars(self, input):
return "".join([i for i in input if ord(i) in range(32, 127)])	return "".join([i for i in input if ord(i) in range(32, 127)])

def getAgencyID(self):	def getAgencyID(self):
""" disclosr agency id """	""" disclosr agency id """
if self.agencyID is None:	if self.agencyID is None:
self.agencyID = os.path.basename(sys.argv[0]).replace(".py", "")	self.agencyID = os.path.basename(sys.argv[0]).replace(".py", "")
return self.agencyID	return self.agencyID

def getURL(self):	def getURL(self):
""" disclog URL"""	""" disclog URL"""
if self.disclogURL is None:	if self.disclogURL is None:
agency = scrape.agencydb.get(self.getAgencyID())	agency = scrape.agencydb.get(self.getAgencyID())
self.disclogURL = agency['FOIDocumentsURL']	self.disclogURL = agency['FOIDocumentsURL']
return self.disclogURL	return self.disclogURL

@abc.abstractmethod	@abc.abstractmethod
def doScrape(self):	def doScrape(self):
""" do the scraping """	""" do the scraping """
return	return


class GenericPDFDisclogScraper(GenericDisclogScraper):	class GenericPDFDisclogScraper(GenericDisclogScraper):

def doScrape(self):	def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,	(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
self.getURL(), "foidocuments", self.getAgencyID())	self.getURL(), "foidocuments", self.getAgencyID())
laparams = LAParams()	laparams = LAParams()
rsrcmgr = PDFResourceManager(caching=True)	rsrcmgr = PDFResourceManager(caching=True)
outfp = StringIO()	outfp = StringIO()
device = TextConverter(rsrcmgr, outfp, codec='utf-8',	device = TextConverter(rsrcmgr, outfp, codec='utf-8',
laparams=laparams)	laparams=laparams)
fp = StringIO()	fp = StringIO()
fp.write(content.read())	fp.write(content.read())

process_pdf(rsrcmgr, device, fp, set(), caching=True,	process_pdf(rsrcmgr, device, fp, set(), caching=True,
check_extractable=True)	check_extractable=True)
description = outfp.getvalue()	description = outfp.getvalue()
fp.close()	fp.close()
device.close()	device.close()
outfp.close()	outfp.close()
dochash = scrape.mkhash(description)	dochash = scrape.mkhash(description)
doc = foidocsdb.get(dochash)	doc = foidocsdb.get(dochash)
if doc is None:	if doc is None:
print "saving " + dochash	print "saving " + dochash
edate = date.today().strftime("%Y-%m-%d")	edate = date.today().strftime("%Y-%m-%d")
doc = {'_id': dochash, 'agencyID': self.getAgencyID()	doc = {'_id': dochash, 'agencyID': self.getAgencyID()
, 'url': self.getURL(), 'docID': dochash,	, 'url': self.getURL(), 'docID': dochash,
"date": edate, "title": "Disclosure Log Updated", "description": description}	"date": edate, "title": "Disclosure Log Updated", "description": description}
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved"	print "already saved"


class GenericDOCXDisclogScraper(GenericDisclogScraper):	class GenericDOCXDisclogScraper(GenericDisclogScraper):

def doScrape(self):	def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url, mime_type, content) = scrape.fetchURL(scrape.docsdb	(url, mime_type, content) = scrape.fetchURL(scrape.docsdb
, self.getURL(), "foidocuments", self.getAgencyID())	, self.getURL(), "foidocuments", self.getAgencyID())
mydoc = zipfile.ZipFile(file)	mydoc = zipfile.ZipFile(file)
xmlcontent = mydoc.read('word/document.xml')	xmlcontent = mydoc.read('word/document.xml')
document = etree.fromstring(xmlcontent)	document = etree.fromstring(xmlcontent)
## Fetch all the text out of the document we just created	## Fetch all the text out of the document we just created
paratextlist = getdocumenttext(document)	paratextlist = getdocumenttext(document)
# Make explicit unicode version	# Make explicit unicode version
newparatextlist = []	newparatextlist = []
for paratext in paratextlist:	for paratext in paratextlist:
newparatextlist.append(paratext.encode("utf-8"))	newparatextlist.append(paratext.encode("utf-8"))
## Print our documnts test with two newlines under each paragraph	## Print our documnts test with two newlines under each paragraph
description = '\n\n'.join(newparatextlist).strip(' \t\n\r')	description = '\n\n'.join(newparatextlist).strip(' \t\n\r')
dochash = scrape.mkhash(description)	dochash = scrape.mkhash(description)
doc = foidocsdb.get(dochash)	doc = foidocsdb.get(dochash)

if doc is None:	if doc is None:
print "saving " + dochash	print "saving " + dochash
edate = time().strftime("%Y-%m-%d")	edate = time().strftime("%Y-%m-%d")
doc = {'_id': dochash, 'agencyID': self.getAgencyID()	doc = {'_id': dochash, 'agencyID': self.getAgencyID()
, 'url': self.getURL(), 'docID': dochash,	, 'url': self.getURL(), 'docID': dochash,
"date": edate, "title": "Disclosure Log Updated", "description": description}	"date": edate, "title": "Disclosure Log Updated", "description": description}
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved"	print "already saved"


class GenericRSSDisclogScraper(GenericDisclogScraper):	class GenericRSSDisclogScraper(GenericDisclogScraper):

def doScrape(self):	def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,	(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
self.getURL(), "foidocuments", self.getAgencyID())	self.getURL(), "foidocuments", self.getAgencyID())
feed = feedparser.parse(content)	feed = feedparser.parse(content)
for entry in feed.entries:	for entry in feed.entries:
#print entry	#print entry
print entry.id	print entry.id
dochash = scrape.mkhash(entry.id)	dochash = scrape.mkhash(entry.id)
doc = foidocsdb.get(dochash)	doc = foidocsdb.get(dochash)
#print doc	#print doc
if doc is None:	if doc is None:
print "saving " + dochash	print "saving " + dochash
edate = datetime.fromtimestamp(	edate = datetime.fromtimestamp(
mktime(entry.published_parsed)).strftime("%Y-%m-%d")	mktime(entry.published_parsed)).strftime("%Y-%m-%d")
doc = {'_id': dochash, 'agencyID': self.getAgencyID(),	doc = {'_id': dochash, 'agencyID': self.getAgencyID(),
'url': entry.link, 'docID': entry.id,	'url': entry.link, 'docID': entry.id,
"date": edate, "title": entry.title}	"date": edate, "title": entry.title}
self.getDescription(entry, entry, doc)	self.getDescription(entry, entry, doc)
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved"	print "already saved"

def getDescription(self, content, entry, doc):	def getDescription(self, content, entry, doc):
""" get description from rss entry"""	""" get description from rss entry"""
doc.update({'description': content.summary})	doc.update({'description': content.summary})
return	return


class GenericOAICDisclogScraper(GenericDisclogScraper):	class GenericOAICDisclogScraper(GenericDisclogScraper):
__metaclass__ = abc.ABCMeta	__metaclass__ = abc.ABCMeta

@abc.abstractmethod	@abc.abstractmethod
def getColumns(self, columns):	def getColumns(self, columns):
""" rearranges columns if required """	""" rearranges columns if required """
return	return

def getColumnCount(self):	def getColumnCount(self):
return 5	return 5

def getDescription(self, content, entry, doc):	def getDescription(self, content, entry, doc):
""" get description from rss entry"""	""" get description from rss entry"""
descriptiontxt = ""	descriptiontxt = ""
for string in content.stripped_strings:	for string in content.stripped_strings:
descriptiontxt = descriptiontxt + " \n" + string	descriptiontxt = descriptiontxt + " \n" + string
doc.update({'description': descriptiontxt})	doc.update({'description': descriptiontxt})

def getTitle(self, content, entry, doc):	def getTitle(self, content, entry, doc):
doc.update({'title': (''.join(content.stripped_strings))})	doc.update({'title': (''.join(content.stripped_strings))})

def getTable(self, soup):	def getTable(self, soup):
return soup.table	return soup.table

def getRows(self, table):	def getRows(self, table):
return table.find_all('tr')	return table.find_all('tr')

def getDate(self, content, entry, doc):	def getDate(self, content, entry, doc):
date = ''.join(content.stripped_strings).strip()	date = ''.join(content.stripped_strings).strip()
(a, b, c) = date.partition("(")	(a, b, c) = date.partition("(")
date = self.remove_control_chars(a.replace("Octber", "October"))	date = self.remove_control_chars(a.replace("Octber", "October"))
print date	print date
edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")	edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
print edate	print edate
doc.update({'date': edate})	doc.update({'date': edate})
return	return

def getLinks(self, content, entry, doc):	def getLinks(self, content, entry, doc):
links = []	links = []
for atag in entry.find_all("a"):	for atag in entry.find_all("a"):
if atag.has_key('href'):	if atag.has_key('href'):
links.append(scrape.fullurl(content, atag['href']))	links.append(scrape.fullurl(content, atag['href']))
if links != []:	if links != []:
doc.update({'links': links})	doc.update({'links': links})
return	return

def doScrape(self):	def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,	(url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
self.getURL(), "foidocuments", self.getAgencyID())	self.getURL(), "foidocuments", self.getAgencyID())
if content is not None:	if content is not None:
if mime_type is "text/html"\	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type == "application/xml":
or mime_type is "application/xhtml+xml"\
or mime_type is"application/xml":
# http://www.crummy.com/software/BeautifulSoup/documentation.html	# http://www.crummy.com/software/BeautifulSoup/documentation.html
	print "parsing"
soup = BeautifulSoup(content)	soup = BeautifulSoup(content)
table = self.getTable(soup)	table = self.getTable(soup)
for row in self.getRows(table):	for row in self.getRows(table):
columns = row.find_all('td')	columns = row.find_all('td')
if len(columns) is self.getColumnCount():	if len(columns) is self.getColumnCount():
(id, date, title,	(id, date, title,
description, notes) = self.getColumns(columns)	description, notes) = self.getColumns(columns)
print self.remove_control_chars(	print self.remove_control_chars(
''.join(id.stripped_strings))	''.join(id.stripped_strings))
if id.string is None:	if id.string is None:
dochash = scrape.mkhash(	dochash = scrape.mkhash(
self.remove_control_chars(	self.remove_control_chars(
url + (''.join(date.stripped_strings))))	url + (''.join(date.stripped_strings))))
else:	else:
dochash = scrape.mkhash(	dochash = scrape.mkhash(
self.remove_control_chars(	self.remove_control_chars(
url + (''.join(id.stripped_strings))))	url + (''.join(id.stripped_strings))))
doc = foidocsdb.get(hash)	doc = foidocsdb.get(dochash)

if doc is None:	if doc is None:
print "saving " + hash	print "saving " + dochash
doc = {'_id': hash,	doc = {'_id': dochash,
'agencyID': self.getAgencyID(),	'agencyID': self.getAgencyID(),
'url': self.getURL(),	'url': self.getURL(),
'docID': (''.join(id.stripped_strings))}	'docID': (''.join(id.stripped_strings))}
self.getLinks(self.getURL(), row, doc)	self.getLinks(self.getURL(), row, doc)
self.getTitle(title, row, doc)	self.getTitle(title, row, doc)
self.getDate(date, row, doc)	self.getDate(date, row, doc)
self.getDescription(description, row, doc)	self.getDescription(description, row, doc)
if notes is not None:	if notes is not None:
doc.update({ 'notes': (	doc.update({ 'notes': (
''.join(notes.stripped_strings))})	''.join(notes.stripped_strings))})
badtitles = ['-','Summary of FOI Request'	badtitles = ['-','Summary of FOI Request'
, 'FOI request(in summary form)'	, 'FOI request(in summary form)'
, 'Summary of FOI request received by the ASC',	, 'Summary of FOI request received by the ASC',
'Summary of FOI request received by agency/minister',	'Summary of FOI request received by agency/minister',
'Description of Documents Requested','FOI request',	'Description of Documents Requested','FOI request',
'Description of FOI Request','Summary of request','Description','Summary',	'Description of FOI Request','Summary of request','Description','Summary',
'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of FOI Request',"FOI request",'Results 1 to 67 of 67']	'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of FOI Request',"FOI request",'Results 1 to 67 of 67']
if doc['title'] not in badtitles\	if doc['title'] not in badtitles\
and doc['description'] != '':	and doc['description'] != '':
print "saving"	print "saving"
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved " + dochash	print "already saved " + dochash

elif len(row.find_all('th')) is self.getColumnCount():	elif len(row.find_all('th')) is self.getColumnCount():
print "header row"	print "header row"

else:	else:
print "ERROR number of columns incorrect"	print "ERROR number of columns incorrect"
print row	print row

file:b/documents/img/feed-icon-14x14.png (new)

 Binary files /dev/null and b/documents/img/feed-icon-14x14.png differ

file:a/documents/index.php -> file:b/documents/index.php

<?php	<?php

include('template.inc.php');	include('template.inc.php');
include_header_documents("");	include_header_documents("");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');
$startkey = (isset($_REQUEST['start_key']) ? $_REQUEST['start_key'] : '9999-99-99');	$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
?>	?>
	<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
	<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
<?php	<?php

$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');

$idtoname = Array();	$idtoname = Array();
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name);	$idtoname[$row->id] = trim($row->value->name);
}	}
$foidocsdb = $server->get_db('disclosr-foidocuments');	$foidocsdb = $server->get_db('disclosr-foidocuments');
try {	try {
$rows = $foidocsdb->get_view("app", "byDate", Array($startkey, '0000-00-00'), true, 20)->rows;	$rows = $foidocsdb->get_view("app", "byDate", Array($endkey, '0000-00-00'), true, 20)->rows;
if ($rows) {	if ($rows) {
foreach ($rows as $key => $row) {	foreach ($rows as $key => $row) {
echo displayLogEntry($row, $idtoname);	echo displayLogEntry($row, $idtoname);
	if (!isset($startkey)) $startkey = $row->key;
$endkey = $row->key;	$endkey = $row->key;
}	}
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
echo "<a href='?start_key=$endkey'>next page</a>";	echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
include_footer_documents();	include_footer_documents();
?>	?>

file:a/documents/rss.xml.php -> file:b/documents/rss.xml.php

<?php	<?php

// Agency X updated Y, new files, diff of plain text/link text,	// Agency X updated Y, new files, diff of plain text/link text,
// feed for just one agency or all	// feed for just one agency or all
// This is a minimum example of using the Universal Feed Generator Class	// This is a minimum example of using the Universal Feed Generator Class
include("../lib/FeedWriter/FeedTypes.php");	include("../lib/FeedWriter/FeedTypes.php");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');
//Creating an instance of FeedWriter class.	//Creating an instance of FeedWriter class.
$TestFeed = new RSS2FeedWriter();	$TestFeed = new RSS2FeedWriter();
//Setting the channel elements	//Setting the channel elements
//Use wrapper functions for common channelelements	////Retriving informations from database
$TestFeed->setTitle('disclosurelo.gs Newest Entries - All');
$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php');
$TestFeed->setDescription('disclosurelo.gs Newest Entries - All Agencies');
$TestFeed->setChannelElement('language', 'en-us');
$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));

//Retriving informations from database
$idtoname = Array();	$idtoname = Array();
$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name);	$idtoname[$row->id] = trim($row->value->name);
}	}
$foidocsdb = $server->get_db('disclosr-foidocuments');	$foidocsdb = $server->get_db('disclosr-foidocuments');
$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99', '0000-00-00', 50), true)->rows;	if (isset($_REQUEST['id'])) {
	$rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows;
	$title = $idtoname[$_REQUEST['id']];
	} else {
	$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99', '0000-00-00', 50), true)->rows;
	$title = 'All Agencies';
	}
	//Use wrapper functions for common channelelements
	$TestFeed->setTitle('disclosurelo.gs Newest Entries - '.$title);
	$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php'.(isset($_REQUEST['id'])? '?id='.$_REQUEST['id'] : ''));
	$TestFeed->setDescription('disclosurelo.gs Newest Entries - '.$title);
	$TestFeed->setChannelElement('language', 'en-us');
	$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));


//print_r($rows);	//print_r($rows);
foreach ($rows as $row) {	foreach ($rows as $row) {
//Create an empty FeedItem	//Create an empty FeedItem
$newItem = $TestFeed->createNewItem();	$newItem = $TestFeed->createNewItem();
//Add elements to the feed item	//Add elements to the feed item
$newItem->setTitle($row->value->title);	$newItem->setTitle($row->value->title);
$newItem->setLink("http://disclosurelo.gs/view.php?id=" . $row->value->_id);	$newItem->setLink("http://disclosurelo.gs/view.php?id=" . $row->value->_id);
$newItem->setDate(strtotime($row->value->date));	$newItem->setDate(strtotime($row->value->date));
$newItem->setDescription(displayLogEntry($row, $idtoname));	$newItem->setDescription(displayLogEntry($row, $idtoname));
$newItem->setAuthor($idtoname[$row->value->agencyID]);	$newItem->setAuthor($idtoname[$row->value->agencyID]);
$newItem->addElement('guid', "http://disclosurelo.gs/view.php?id=" . $row->value->_id, array('isPermaLink' => 'true'));	$newItem->addElement('guid', "http://disclosurelo.gs/view.php?id=" . $row->value->_id, array('isPermaLink' => 'true'));
//Now add the feed item	//Now add the feed item
$TestFeed->addItem($newItem);	$TestFeed->addItem($newItem);
}	}
//OK. Everything is done. Now genarate the feed.	//OK. Everything is done. Now genarate the feed.
$TestFeed->generateFeed();	$TestFeed->generateFeed();
?>	?>

file:b/documents/scrapers/00a294de663db69062ca09aede7c0487.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import dateutil
	from dateutil.parser import *
	from datetime import *


	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):

	def __init__(self):
	super(ScraperImplementation, self).__init__()
	def getDate(self, content, entry, doc):
	date = ''.join(entry.find('th').stripped_strings).strip()
	(a, b, c) = date.partition("(")
	date = self.remove_control_chars(a.replace("Octber", "October"))
	print date
	edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
	print edate
	doc.update({'date': edate})
	return
	def getColumnCount(self):
	return 4

	def getTable(self, soup):
	return soup.find(summary="List of Defence documents released under Freedom of Information requets")

	def getColumns(self, columns):
	(id, description, access, notes) = columns
	return (id, None, description, description, notes)


	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)

	nsi = ScraperImplementation()
	nsi.disclogURL = "http://www.defence.gov.au/foi/disclosure_log_201213.cfm"
	nsi.doScrape()

	nsi.disclogURL = "http://www.defence.gov.au/foi/disclosure_log_201112.cfm"
	nsi.doScrape()

	nsi.disclogURL = "http://www.defence.gov.au/foi/disclosure_log_201011.cfm"
	nsi.doScrape()

file:a/documents/scrapers/00a294de663db69062ca09aede7c0487.txt (deleted)

multipage

file:a/documents/scrapers/0ae822d1a748e60d90f0b79b97d5a3e5.txt (deleted)

ACMA style

file:b/documents/scrapers/1803322b27286950cab0c543168b5f21.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import dateutil
	from dateutil.parser import *
	from datetime import *
	import scrape
	from bs4 import BeautifulSoup
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):

	def __init__(self):
	super(ScraperImplementation, self).__init__()

	def getDescription(self,content, entry,doc):
	link = None
	links = []
	description = ""
	for atag in entry.find_all('a'):
	if atag.has_key('href'):
	link = scrape.fullurl(self.getURL(), atag['href'])
	(url, mime_type, htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
	if htcontent != None:
	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
	soup = BeautifulSoup(htcontent)
	row = soup.find(id="content_div_148050")
	description = ''.join(row.stripped_strings)
	for atag in row.find_all("a"):
	if atag.has_key('href'):
	links.append(scrape.fullurl(link, atag['href']))

	if links != []:
	doc.update({'links': links})
	if description != "":
	doc.update({ 'description': description})
	def getColumnCount(self):
	return 4

	def getColumns(self, columns):
	(id, date, datepub, title) = columns
	return (id, date, title, title, None)


	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)

	nsi = ScraperImplementation()
	nsi.disclogURL = "http://www.dbcde.gov.au/about_us/freedom_of_information_disclosure_log/foi_list?result_146858_result_page=1"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dbcde.gov.au/about_us/freedom_of_information_disclosure_log/foi_list?result_146858_result_page=2"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dbcde.gov.au/about_us/freedom_of_information_disclosure_log/foi_list?result_146858_result_page=3"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dbcde.gov.au/about_us/freedom_of_information_disclosure_log/foi_list?result_146858_result_page=4"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dbcde.gov.au/about_us/freedom_of_information_disclosure_log/foi_list?result_146858_result_page=5"
	nsi.doScrape()

file:a/documents/scrapers/1803322b27286950cab0c543168b5f21.txt (deleted)

multipage log

file:b/documents/scrapers/8796220032faf94501bd366763263685.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import dateutil
	from dateutil.parser import *
	from datetime import *


	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):

	def __init__(self):
	super(ScraperImplementation, self).__init__()

	def getColumnCount(self):
	return 6

	def getColumns(self, columns):
	(id, date, title, description, datepub, notes) = columns
	return (id, date, title, description, notes)


	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)

	nsi = ScraperImplementation()
	nsi.disclogURL = "http://www.dpmc.gov.au/foi/ips/disclosure_logs/pmo/2011-12.cfm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dpmc.gov.au/foi/ips/disclosure_logs/dpmc/2011-12.cfm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dpmc.gov.au/foi/ips/disclosure_logs/dpmc/2012-13.cfm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dpmc.gov.au/foi/ips/disclosure_logs/omsi/2011-12.cfm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.dpmc.gov.au/foi/ips/disclosure_logs/omps/2012-13.cfm"
	nsi.doScrape()

file:a/documents/scrapers/8796220032faf94501bd366763263685.txt (deleted)

multiple pages

file:a/documents/scrapers/8e874a2fde8aa0ccdc6d14573d766540.py -> file:b/documents/scrapers/8e874a2fde8aa0ccdc6d14573d766540.py

import sys,os	import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers	import genericScrapers
import scrape	import scrape
from bs4 import BeautifulSoup	from bs4 import BeautifulSoup
import codecs	import codecs
#http://www.doughellmann.com/PyMOTW/abc/	#http://www.doughellmann.com/PyMOTW/abc/
class NewScraperImplementation(genericScrapers.GenericOAICDisclogScraper):	class NewScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
def getDescription(self,content, entry,doc):	def getDescription(self,content, entry,doc):
link = None	link = None
links = []	links = []
description = ""	description = ""
for atag in entry.find_all('a'):	for atag in entry.find_all('a'):
if atag.has_key('href'):	if atag.has_key('href'):
link = scrape.fullurl(self.getURL(),atag['href'])	link = scrape.fullurl(self.getURL(),atag['href'])
(url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)	(url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
if htcontent != None:	if htcontent != None:
if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
# http://www.crummy.com/software/BeautifulSoup/documentation.html	# http://www.crummy.com/software/BeautifulSoup/documentation.html
soup = BeautifulSoup(htcontent)	soup = BeautifulSoup(htcontent)
for text in soup.find(id="divFullWidthColumn").stripped_strings:	for text in soup.find(id="divFullWidthColumn").stripped_strings:
description = description + text.encode('ascii', 'ignore')	description = description + text.encode('ascii', 'ignore')

for atag in soup.find(id="divFullWidthColumn").find_all("a"):	for atag in soup.find(id="divFullWidthColumn").find_all("a"):
if atag.has_key('href'):	if atag.has_key('href'):
links.append(scrape.fullurl(link,atag['href']))	links.append(scrape.fullurl(link,atag['href']))

if links != []:	if links != []:
doc.update({'links': links})	doc.update({'links': links})
if description != "":	if description != "":
doc.update({ 'description': description})	doc.update({ 'description': description})

def getColumnCount(self):	def getColumnCount(self):
return 2	return 2
def getTable(self,soup):	def getTable(self,soup):
return soup.find(id = "TwoColumnSorting")	return soup.find(id = "TwoColumnSorting")
def getColumns(self,columns):	def getColumns(self,columns):
( title, date) = columns	( title, date) = columns
return (title, date, title, title, None)	return (title, date, title, title, None)
class OldScraperImplementation(genericScrapers.GenericOAICDisclogScraper):	class OldScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
def getDescription(self,content, entry,doc):	def getDescription(self,content, entry,doc):
link = None	link = None
links = []	links = []
description = ""	description = ""
for atag in entry.find_all('a'):	for atag in entry.find_all('a'):
if atag.has_key('href'):	if atag.has_key('href'):
link = scrape.fullurl(self.getURL(),atag['href'])	link = scrape.fullurl(self.getURL(),atag['href'])
(url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)	(url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
if htcontent != None:	if htcontent != None:
if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
# http://www.crummy.com/software/BeautifulSoup/documentation.html	# http://www.crummy.com/software/BeautifulSoup/documentation.html
soup = BeautifulSoup(htcontent)	soup = BeautifulSoup(htcontent)
for text in soup.find(id="content-item").stripped_strings:	for text in soup.find(id="content-item").stripped_strings:
description = description + text + " \n"	description = description + text + " \n"
for atag in soup.find(id="content-item").find_all("a"):	for atag in soup.find(id="content-item").find_all("a"):
if atag.has_key('href'):	if atag.has_key('href'):
links.append(scrape.fullurl(link,atag['href']))	links.append(scrape.fullurl(link,atag['href']))
if links != []:	if links != []:
doc.update({'links': links})	doc.update({'links': links})
if description != "":	if description != "":
doc.update({ 'description': description})	doc.update({ 'description': description})

if links != []:	if links != []:
doc.update({'links': links})	doc.update({'links': links})
if description != "":	if description != "":
doc.update({ 'description': description})	doc.update({ 'description': description})

def getColumnCount(self):	def getColumnCount(self):
return 2	return 2
def getTable(self,soup):	def getTable(self,soup):
return soup.find(class_ = "doc-list")	return soup.find(class_ = "doc-list")
def getColumns(self,columns):	def getColumns(self,columns):
(date, title) = columns	(date, title) = columns
return (title, date, title, title, None)	return (title, date, title, title, None)

if __name__ == '__main__':	if __name__ == '__main__':
print 'Subclass:', issubclass(NewScraperImplementation, genericScrapers.GenericOAICDisclogScraper)	print 'Subclass:', issubclass(NewScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
print 'Instance:', isinstance(NewScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)	print 'Instance:', isinstance(NewScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
#NewScraperImplementation().doScrape()	NewScraperImplementation().doScrape()
print 'Subclass:', issubclass(OldScraperImplementation, genericScrapers.GenericOAICDisclogScraper)	print 'Subclass:', issubclass(OldScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
print 'Instance:', isinstance(OldScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)	print 'Instance:', isinstance(OldScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
osi = OldScraperImplementation()	osi = OldScraperImplementation()
osi.disclogURL = "http://archive.treasury.gov.au/content/foi_publications.asp?year=-1&abstract=0&classification=&=&titl=Disclosure+Log+-+Documents+Released+Under+FOI"	osi.disclogURL = "http://archive.treasury.gov.au/content/foi_publications.asp?year=-1&abstract=0&classification=&=&titl=Disclosure+Log+-+Documents+Released+Under+FOI"
osi.doScrape()	osi.doScrape()
# old site too

file:b/documents/scrapers/b0a3281ba66efe173c5a33d5ef90ff76.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import dateutil
	from dateutil.parser import *
	from datetime import *


	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):

	def __init__(self):
	super(ScraperImplementation, self).__init__()

	def getColumnCount(self):
	return 2

	def getColumns(self, columns):
	(date, title) = columns
	return (title, date, title, title, None)


	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)

	nsi = ScraperImplementation()
	nsi.disclogURL = "http://www.immi.gov.au/about/foi/foi-disclosures-2012.htm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.immi.gov.au/about/foi/foi-disclosures-2011.htm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.immi.gov.au/about/foi/foi-disclosures-2010.htm"
	nsi.doScrape()
	nsi.disclogURL = "http://www.immi.gov.au/about/foi/foi-disclosures-2009.htm"
	nsi.doScrape()

file:a/documents/scrapers/b0a3281ba66efe173c5a33d5ef90ff76.txt (deleted)

multipage immi

file:a/documents/sitemap.xml.php -> file:b/documents/sitemap.xml.php

<?php	<?php

include ('../include/common.inc.php');	include ('../include/common.inc.php');
$last_updated = date('Y-m-d', @filemtime('cbrfeed.zip'));	$last_updated = date('Y-m-d', @filemtime('cbrfeed.zip'));
header("Content-Type: text/xml");	header("Content-Type: text/xml");
echo "<?xml version='1.0' encoding='UTF-8'?>";	echo "<?xml version='1.0' encoding='UTF-8'?>";
echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";	echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
echo " <url><loc>" . local_url() . "index.php</loc><priority>1.0</priority></url>\n";	echo " <url><loc>" . local_url() . "index.php</loc><priority>1.0</priority></url>\n";
foreach (scandir("./") as $file) {	foreach (scandir("./") as $file) {
if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php")	if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php")
echo " <url><loc>" . local_url() . "$file</loc><priority>0.6</priority></url>\n";	echo " <url><loc>" . local_url() . "$file</loc><priority>0.6</priority></url>\n";
}	}
	$agenciesdb = $server->get_db('disclosr-agencies');
$db = $server->get_db('disclosr-foidocuments');
try {	try {
$rows = $db->get_view("app", "all")->rows;	$rows = $agenciesdb->get_view("app", "byCanonicalName")->rows;
	foreach ($rows as $row) {
	echo '<url><loc>' . local_url() . 'agency.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
	}
	} catch (SetteeRestClientException $e) {
	setteErrorHandler($e);
	}
	$foidocsdb = $server->get_db('disclosr-foidocuments');
	try {
	$rows = $foidocsdb->get_view("app", "all")->rows;
foreach ($rows as $row) {	foreach ($rows as $row) {
echo '<url><loc>' . local_url() . 'view.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";	echo '<url><loc>' . local_url() . 'view.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
echo '</urlset>';	echo '</urlset>';
?>	?>

file:a/documents/template.inc.php -> file:b/documents/template.inc.php

<?php	<?php

function include_header_documents($title) {	function include_header_documents($title) {
header('X-UA-Compatible: IE=edge,chrome=1');	header('X-UA-Compatible: IE=edge,chrome=1');
?>	?>
<!doctype html>	<!doctype html>
<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->	<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->	<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->	<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->	<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->	<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->	<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
<head>	<head>
<meta charset="utf-8">	<meta charset="utf-8">

<title>Australian Disclosure Logs<?php if ($title != "") echo " - $title"; ?></title>	<title>Australian Disclosure Logs<?php if ($title != "") echo " - $title"; ?></title>
<meta name="description" content="">	<meta name="description" content="">

<!-- Mobile viewport optimized: h5bp.com/viewport -->	<!-- Mobile viewport optimized: h5bp.com/viewport -->
<meta name="viewport" content="width=device-width">	<meta name="viewport" content="width=device-width">
<link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />	<link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />
<!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->	<!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />	<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />

<!-- Le styles -->	<!-- Le styles -->
<link href="css/bootstrap.min.css" rel="stylesheet">	<link href="css/bootstrap.min.css" rel="stylesheet">
<style type="text/css">	<style type="text/css">
body {	body {
padding-top: 60px;	padding-top: 60px;
padding-bottom: 40px;	padding-bottom: 40px;
}	}
.sidebar-nav {	.sidebar-nav {
padding: 9px 0;	padding: 9px 0;
}	}
</style>	</style>
<link href="css/bootstrap-responsive.min.css" rel="stylesheet">	<link href="css/bootstrap-responsive.min.css" rel="stylesheet">

<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->	<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
<!--[if lt IE 9]>	<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>	<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->	<![endif]-->
<!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->	<!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->

<!-- All JavaScript at the bottom, except this Modernizr build.	<!-- All JavaScript at the bottom, except this Modernizr build.
Modernizr enables HTML5 elements & feature detects for optimal performance.	Modernizr enables HTML5 elements & feature detects for optimal performance.
Create your own custom Modernizr build: www.modernizr.com/download/	Create your own custom Modernizr build: www.modernizr.com/download/
<script src="js/libs/modernizr-2.5.3.min.js"></script>-->	<script src="js/libs/modernizr-2.5.3.min.js"></script>-->
<script src="js/jquery.js"></script>	<script src="js/jquery.js"></script>
<script type="text/javascript" src="js/flotr2.min.js"></script>	<script type="text/javascript" src="js/flotr2.min.js"></script>

</head>	</head>
<body>	<body>
<div class="navbar navbar-inverse navbar-fixed-top">	<div class="navbar navbar-inverse navbar-fixed-top">
<div class="navbar-inner">	<div class="navbar-inner">
<div class="container-fluid">	<div class="container-fluid">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">	<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>	<span class="icon-bar"></span>
<span class="icon-bar"></span>	<span class="icon-bar"></span>
<span class="icon-bar"></span>	<span class="icon-bar"></span>
</a>	</a>
<a class="brand" href="#">Australian Disclosure Logs</a>	<a class="brand" href="#">Australian Disclosure Logs</a>
<div class="nav-collapse collapse">	<div class="nav-collapse collapse">
<p class="navbar-text pull-right">	<p class="navbar-text pull-right">
Check out our subsites on:	Check out our subsites on:
<a href="http://orgs.disclosurelo.gs">Government Agencies</a>	<a href="http://orgs.disclosurelo.gs">Government Agencies</a>
• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>	• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>
• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>	• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>

</p>	</p>
<ul class="nav">	<ul class="nav">
<li><a href="index.php">Home</a></li>	<li><a href="agency.php">By Agency</a></li>
	<li><a href="date.php">By Date</a></li>
<li><a href="disclogsList.php">List of Disclosure Logs</a></li>	<li><a href="disclogsList.php">List of Disclosure Logs</a></li>
<li><a href="about.php">About</a></li>	<li><a href="about.php">About</a></li>

</ul>	</ul>
</div><!--/.nav-collapse -->	</div><!--/.nav-collapse -->
</div>	</div>
</div>	</div>
</div>	</div>
<div class="container">	<div class="container">
<?php	<?php
}	}

function include_footer_documents() {	function include_footer_documents() {
?>	?>
</div> <!-- /container -->	</div> <!-- /container -->
<hr>	<hr>

<footer>	<footer>
<p>Not affiliated with or endorsed by any government agency.</p>	<p>Not affiliated with or endorsed by any government agency.</p>
</footer>	</footer>
<script type="text/javascript">	<?php
	if ($ENV != "DEV") {
	echo "<script type='text/javascript'>

var _gaq = _gaq \|\| [];	var _gaq = _gaq \|\| [];
_gaq.push(['_setAccount', 'UA-12341040-4']);	_gaq.push(['_setAccount', 'UA-12341040-4']);
_gaq.push(['_setDomainName', 'disclosurelo.gs']);	_gaq.push(['_setDomainName', 'disclosurelo.gs']);
_gaq.push(['_setAllowLinker', true]);	_gaq.push(['_setAllowLinker', true]);
_gaq.push(['_trackPageview']);	_gaq.push(['_trackPageview']);

(function() {	(function() {
var ga = document.createElement('script');	var ga = document.createElement('script');
ga.type = 'text/javascript';	ga.type = 'text/javascript';
ga.async = true;	ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';	ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0];	var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);	s.parentNode.insertBefore(ga, s);
})();	})();

</script>	</script>";
	}
	?>
<!-- Le javascript	<!-- Le javascript
================================================== -->	================================================== -->
<!-- Placed at the end of the document so the pages load faster -->	<!-- Placed at the end of the document so the pages load faster -->
<!--	<!--
<script src="js/bootstrap-transition.js"></script>	<script src="js/bootstrap-transition.js"></script>
<script src="js/bootstrap-alert.js"></script>	<script src="js/bootstrap-alert.js"></script>
<script src="js/bootstrap-modal.js"></script>	<script src="js/bootstrap-modal.js"></script>
<script src="js/bootstrap-dropdown.js"></script>	<script src="js/bootstrap-dropdown.js"></script>
<script src="js/bootstrap-scrollspy.js"></script>	<script src="js/bootstrap-scrollspy.js"></script>
<script src="js/bootstrap-tab.js"></script>	<script src="js/bootstrap-tab.js"></script>
<script src="js/bootstrap-tooltip.js"></script>	<script src="js/bootstrap-tooltip.js"></script>
<script src="js/bootstrap-popover.js"></script>	<script src="js/bootstrap-popover.js"></script>
<script src="js/bootstrap-button.js"></script>	<script src="js/bootstrap-button.js"></script>
<script src="js/bootstrap-collapse.js"></script>	<script src="js/bootstrap-collapse.js"></script>
<script src="js/bootstrap-carousel.js"></script>	<script src="js/bootstrap-carousel.js"></script>
<script src="js/bootstrap-typeahead.js"></script>-->	<script src="js/bootstrap-typeahead.js"></script>-->


</body>	</body>
</html>	</html>
<?php	<?php
}	}

function truncate($string, $length, $stopanywhere = false) {	function truncate($string, $length, $stopanywhere = false) {
//truncates a string to a certain char length, stopping on a word if not specified otherwise.	//truncates a string to a certain char length, stopping on a word if not specified otherwise.
if (strlen($string) > $length) {	if (strlen($string) > $length) {
//limit hit!	//limit hit!
$string = substr($string, 0, ($length - 3));	$string = substr($string, 0, ($length - 3));
if ($stopanywhere) {	if ($stopanywhere) {
//stop anywhere	//stop anywhere
$string .= '...';	$string .= '...';
} else {	} else {
//stop on a word.	//stop on a word.
$string = substr($string, 0, strrpos($string, ' ')) . '...';	$string = substr($string, 0, strrpos($string, ' ')) . '...';
}	}
}	}
return $string;	return $string;
}	}

function displayLogEntry($row, $idtoname) {	function displayLogEntry($row, $idtoname) {
$result = "";	$result = "";
$result .= '<div itemscope itemtype="http://schema.org/Article">';	$result .= '<div itemscope itemtype="http://schema.org/Article">';
$result .= '<h2> <span itemprop="datePublished">' . $row->value->date . "</span>: <span itemprop='name headline'>" . truncate($row->value->title, 120) . "</span>";	$result .= '<h2> <span itemprop="datePublished">' . $row->value->date . "</span>: <span itemprop='name headline'>" . truncate($row->value->title, 120) . "</span>";
$result .= ' (<span itemprop="author publisher creator">' . $idtoname[$row->value->agencyID] . '</span>)</h2>';	$result .= ' (<span itemprop="author publisher creator">' . $idtoname[$row->value->agencyID] . '</span>)</h2>';
$result .= "<p itemprop='description articleBody text'> Title: " . $row->value->title . "<br/>";	$result .= "<p itemprop='description articleBody text'> Title: " . $row->value->title . "<br/>";
if (isset($row->value->description)) {	if (isset($row->value->description)) {
$result .= str_replace("\n", "<br>", preg_replace("/(^[\r\n]\|[\r\n]+)[\s\t][\r\n]+/", "",trim($row->value->description)));	$result .= str_replace("\n", "<br>", preg_replace("/(^[\r\n]\|[\r\n]+)[\s\t][\r\n]+/", "",trim($row->value->description)));
}	}
if (isset($row->value->notes)) {	if (isset($row->value->notes)) {
$result .= " <br>Note: " . $row->value->notes;	$result .= " <br>Note: " . $row->value->notes;
}	}
$result .= "</p>";	$result .= "</p>";

if (isset($row->value->links)) {	if (isset($row->value->links)) {
$result .= '<h3>Links/Documents</h3><ul itemprop="associatedMedia">';	$result .= '<h3>Links/Documents</h3><ul itemprop="associatedMedia">';
foreach ($row->value->links as $link) {	foreach ($row->value->links as $link) {
$result .= '<li itemscope itemtype="http://schema.org/MediaObject"><a href=' . $link . ' itemprop="url contentURL">' . urlencode($link) . "</a></li>";	$result .= '<li itemscope itemtype="http://schema.org/MediaObject"><a href=' . htmlspecialchars ($link) . ' itemprop="url contentURL">' . htmlspecialchars ( $link) . "</a></li>";
}	}

$result .= "</ul>";	$result .= "</ul>";
}	}
$result .= "<small><A itemprop='url' href='" . $row->value->url . "'>View original source...</a> ID: " . strip_tags($row->value->docID) . "</small>";	$result .= "<small><A itemprop='url' href='" . $row->value->url . "'>View original source...</a> ID: " . strip_tags($row->value->docID) . "</small>";
$result .= "</div>";	$result .= "</div>\n";
return $result;	return $result;
}	}

file:a/documents/view.php -> file:b/documents/view.php

<?php	<?php
include('template.inc.php');	include('template.inc.php');
include_header_documents("");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');
?>	?>
<?php	<?php



$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');

$idtoname = Array();	$idtoname = Array();
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {	foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name);	$idtoname[$row->id] = trim($row->value->name);
}	}
$foidocsdb = $server->get_db('disclosr-foidocuments');	$foidocsdb = $server->get_db('disclosr-foidocuments');
try {	try {
$obj = new stdClass();	$obj = new stdClass();
$obj->value = $foidocsdb->get($_REQUEST['id']);	$obj->value = $foidocsdb->get($_REQUEST['id']);
	include_header_documents($obj->value->title);

echo displayLogEntry($obj,$idtoname);	echo displayLogEntry($obj,$idtoname);

} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
include_footer_documents();	include_footer_documents();
?>	?>