RTK import
Former-commit-id: 831a25e0eea93541a7ab3816694f3feeda047778
--- a/admin/genericAgencyFixer.php
+++ b/admin/genericAgencyFixer.php
@@ -7,28 +7,48 @@
$db = $server->get_db('disclosr-agencies');
+// metatags
+try {
+ $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+ //print_r($rows);
+ foreach ($agencies as $agency) {
+ if (isset($agency->value->scrapeDepth)) {
+ unset($agency->value->scrapeDepth);
+ }
+ if (isset($agency->value->lastScraped)) {
+ unset($agency->value->lastScraped);
+ }
+ $db->save($agency->value);
+ echo "<hr>";
+ flush();
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+// metatags
try {
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows);
foreach ($agencies as $agency) {
//echo $agency->value->name . " ".$agency->value->website."<br />\n";
- // print_r($agency);
+ // print_r($agency);
//hasRestricitiveLicence" hasRestrictiveLicense -> has Restrictive Licence
// "hasYoutube" -> Tube
// "comment" -> "comments"
if (!isset($agency->value->metaTags) && isset($agency->value->website)) {
- echo $agency->value->name . " ".$agency->value->website."<br />\n";
+ echo $agency->value->name . " " . $agency->value->website . "<br />\n";
$agency->value->metaTags = Array();
$request = Requests::get($agency->value->website);
$html = phpQuery::newDocumentHTML($request->body);
phpQuery::selectDocument($html);
foreach (pq('meta')->elements as $meta) {
- $tagName = $meta->getAttribute('name');;
+ $tagName = $meta->getAttribute('name');
+ ;
$content = $meta->getAttribute('content');
if ($tagName != "") {
-echo "$tagName == $content <br>\n";
- $agency->value->metaTags[$tagName] = $content;
+ echo "$tagName == $content <br>\n";
+ $agency->value->metaTags[$tagName] = $content;
}
}
//print_r($agency->value->metaTags);
--- a/admin/importRTKbodies.php
+++ b/admin/importRTKbodies.php
@@ -29,6 +29,7 @@
} else {
echo $Row[array_search($nameField, $headers)] . PHP_EOL;
$accounts[$nametoid[trim($agencyName)]]["rtkURLs"][$agencyName] = 'http://www.righttoknow.org.au/body/'.$Row[array_search($accountField, $headers)];
+ $accounts[$nametoid[trim($agencyName)]]["rtkDescriptions"][$agencyName] = $Row[array_search("Notes", $headers)];
}
} else {
echo "error finding any agency" . $line . PHP_EOL;
@@ -38,19 +39,26 @@
}
extractCSVAccounts("http://www.righttoknow.org.au/body/all-authorities.csv","Agency","URL name");
-print_r($accounts);
-/* foreach ($accounts as $id => $accountTypes) {
+//print_r($accounts);
+ foreach ($accounts as $id => $allvalues) {
echo $id . "<br>" . PHP_EOL;
$doc = object_to_array($db->get($id));
// print_r($doc);
- foreach ($accountTypes as $accountType => $accounts) {
- if (!isset($doc["has" . $accountType]) || !is_array($doc["has" . $accountType])) {
- $doc["has" . $accountType] = Array();
+ foreach ($allvalues as $valueType => $values) {
+ if (!isset($doc[ $valueType]) || !is_array($doc[ $valueType])) {
+ $doc[ $valueType] = Array();
}
- $doc["has" . $accountType] = array_unique(array_merge($doc["has" . $accountType], $accounts));
+ $doc[ $valueType] = array_unique(array_merge($doc[ $valueType], $values));
+ if ( $valueType == "rtkDescriptions") {
+ foreach ($values as $descriptionAgency => $descriptionValue) {
+ if ($descriptionAgency == $doc->value->name) {
+ $doc->value->description = $descriptionValue;
+ }
+ }
+ }
}
$db->save($doc);
-}*/
+}
?>
--- a/couchdb/settee/src/classes/SetteeDatabase.class.php
+++ b/couchdb/settee/src/classes/SetteeDatabase.class.php
@@ -1,310 +1,316 @@
<?php
/**
-* Databaase class.
-*/
+ * Databaase class.
+ */
class SetteeDatabase {
- /**
- * Base URL of the CouchDB REST API
- */
- private $conn_url;
-
- /**
- * HTTP REST Client instance
- */
- protected $rest_client;
-
- /**
- * Name of the database
- */
- private $dbname;
-
- /**
- * Default constructor
- */
- function __construct($conn_url, $dbname) {
- $this->conn_url = $conn_url;
- $this->dbname = $dbname;
- $this->rest_client = SetteeRestClient::get_instance($this->conn_url);
- }
-
-
- /**
- * Get UUID from CouchDB
- *
- * @return
- * CouchDB-generated UUID string
- *
- */
- function gen_uuid() {
- $ret = $this->rest_client->http_get('_uuids');
- return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking
- }
-
- /**
- * Create or update a document database
- *
- * @param $document
- * PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.
- *
- * <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).
- * If "_id" is missing, CouchDB will be used to generate a UUID.
- *
- * <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.
- * You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be
- * one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but
- * not provide "_id" since that is an invalid input.
- *
- * @param $allowRevAutoDetection
- * Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision
- * for a document and use it. This option is "false" by default because it involves an extra http HEAD request and
- * therefore can make save() operation slightly slower if such auto-detection is not required.
- *
- * @return
- * document object with the database id (uuid) and revision attached;
- *
- * @throws SetteeCreateDatabaseException
- */
- function save($document, $allowRevAutoDetection = false) {
- if (is_string($document)) {
- $document = json_decode($document);
- }
-
- // Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)
- if(is_array($document)) {
- $document = (object) $document;
- }
-
- if (empty($document->_id) && empty($document->_rev)) {
- $id = $this->gen_uuid();
- }
- elseif (empty($document->_id) && !empty($document->_rev)) {
- throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");
- }
- else {
- $id = $document->_id;
-
- if ($allowRevAutoDetection) {
- try {
- $rev = $this->get_rev($id);
- } catch (SetteeRestClientException $e) {
- // auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error
- }
- if (!empty($rev)) {
- $document->_rev = $rev;
- }
- }
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $document_json = json_encode($document, JSON_NUMERIC_CHECK);
-
- $ret = $this->rest_client->http_put($full_uri, $document_json);
-
- $document->_id = $ret['decoded']->id;
- $document->_rev = $ret['decoded']->rev;
-
- return $document;
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $content
- * Content of the attachment in a string-buffer format. This function will automatically base64-encode content for
- * you, so you don't have to do it.
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment($doc, $name, $content, $mime_type = null) {
- if (empty($doc->_attachments) || !is_object($doc->_attachments)) {
- $doc->_attachments = new stdClass();
- }
-
- if (empty($mime_type)) {
- $mime_type = $this->rest_client->content_mime_type($content);
- }
-
- $doc->_attachments->$name = new stdClass();
- $doc->_attachments->$name->content_type = $mime_type;
- $doc->_attachments->$name->data = base64_encode($content);
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $file
- * Full path to a file (e.g. as returned by PHP's realpath function).
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment_file($doc, $name, $file, $mime_type = null) {
- $content = file_get_contents($file);
- $this->add_attachment($doc, $name, $content, $mime_type);
- }
-
- /**
- *
- * Retrieve a document from CouchDB
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri);
- $ret = $this->rest_client->http_get($full_uri);
- return $ret['decoded'];
- }
-
- /**
- *
- * Get the latest revision of a document with document id: $id in CouchDB.
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get_rev($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $headers = $this->rest_client->http_head($full_uri);
- if (empty($headers['Etag'])) {
- throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");
- }
- $etag = str_replace('"', '', $headers['Etag']);
- return $etag;
- }
-
- /**
- * Delete a document
- *
- * @param $document
- * a PHP object or JSON representation of the document that has _id and _rev fields.
- *
- * @return void
- */
- function delete($document) {
- if (!is_object($document)) {
- $document = json_decode($document);
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;
- $this->rest_client->http_delete($full_uri);
- }
-
-
- /*----------------- View-related functions --------------*/
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $map_src
- * Source code of the map function in Javascript
- * @param $reduce_src
- * Source code of the reduce function in Javascript (optional)
- * @return void
- */
- function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $obj->views->$view_name->map = $map_src;
- if (!empty($reduce_src)) {
- $obj->views->$view_name->reduce = $reduce_src;
- }
-
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- return $this->save($obj, true);
- }
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $key
- * key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes
- * that first element is startkey, second: endkey.
- * @param $descending
- * return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change
- * order you also need to swap startkey and endkey values!
- *
- * @return void
- */
- function get_view($design_doc, $view_name, $key = null, $descending = false, $limit = false) {
- $id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $id .= "/_view/$view_name";
-
- $data = array();
- if (!empty($key)) {
- if (is_string($key)) {
- $data = "key=" . '"' . $key . '"';
- }
- elseif (is_array($key)) {
- list($startkey, $endkey) = $key;
- $data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';
- }
-
- if ($descending) {
- $data .= "&descending=true";
- }
- if ($limit) {
- $data .= "&limit=".$limit;
- }
- }
-
-
-
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%253Fgroup%253D","?group=",$full_uri);
-$full_uri = str_replace("%253Flimit%253D","?limit=",$full_uri);
- $ret = $this->rest_client->http_get($full_uri, $data);
- return $ret['decoded'];
-
- }
-
- /**
- * @param $id
- * @return
- * return a properly url-encoded id.
- */
- private function safe_urlencode($id) {
- //-- System views like _design can have "/" in their URLs.
- $id = rawurlencode($id);
- if (substr($id, 0, 1) == '_') {
- $id = str_replace('%2F', '/', $id);
- }
- return $id;
- }
-
- /** Getter for a database name */
- function get_name() {
- return $this->dbname;
- }
+ /**
+ * Base URL of the CouchDB REST API
+ */
+ private $conn_url;
+
+ /**
+ * HTTP REST Client instance
+ */
+ protected $rest_client;
+
+ /**
+ * Name of the database
+ */
+ private $dbname;
+
+ /**
+ * Default constructor
+ */
+ function __construct($conn_url, $dbname) {
+ $this->conn_url = $conn_url;
+ $this->dbname = $dbname;
+ $this->rest_client = SetteeRestClient::get_instance($this->conn_url);
+ }
+
+ /**
+ * Get UUID from CouchDB
+ *
+ * @return
+ * CouchDB-generated UUID string
+ *
+ */
+ function gen_uuid() {
+ $ret = $this->rest_client->http_get('_uuids');
+ return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking
+ }
+
+ /**
+ * Create or update a document database
+ *
+ * @param $document
+ * PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.
+ *
+ * <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).
+ * If "_id" is missing, CouchDB will be used to generate a UUID.
+ *
+ * <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.
+ * You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be
+ * one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but
+ * not provide "_id" since that is an invalid input.
+ *
+ * @param $allowRevAutoDetection
+ * Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision
+ * for a document and use it. This option is "false" by default because it involves an extra http HEAD request and
+ * therefore can make save() operation slightly slower if such auto-detection is not required.
+ *
+ * @return
+ * document object with the database id (uuid) and revision attached;
+ *
+ * @throws SetteeCreateDatabaseException
+ */
+ function save($document, $allowRevAutoDetection = false) {
+ if (is_string($document)) {
+ $document = json_decode($document);
+ }
+
+ // Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)
+ if (is_array($document)) {
+ $document = (object) $document;
+ }
+
+ if (empty($document->_id) && empty($document->_rev)) {
+ $id = $this->gen_uuid();
+ } elseif (empty($document->_id) && !empty($document->_rev)) {
+ throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");
+ } else {
+ $id = $document->_id;
+
+ if ($allowRevAutoDetection) {
+ try {
+ $rev = $this->get_rev($id);
+ } catch (SetteeRestClientException $e) {
+ // auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error
+ }
+ if (!empty($rev)) {
+ $document->_rev = $rev;
+ }
+ }
+ }
+
+ $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
+ $document_json = json_encode($document, JSON_NUMERIC_CHECK);
+
+ $ret = $this->rest_client->http_put($full_uri, $document_json);
+
+ $document->_id = $ret['decoded']->id;
+ $document->_rev = $ret['decoded']->rev;
+
+ return $document;
+ }
+
+ /**
+ * @param $doc
+ * @param $name
+ * @param $content
+ * Content of the attachment in a string-buffer format. This function will automatically base64-encode content for
+ * you, so you don't have to do it.
+ * @param $mime_type
+ * Optional. Will be auto-detected if not provided
+ * @return void
+ */
+ public function add_attachment($doc, $name, $content, $mime_type = null) {
+ if (empty($doc->_attachments) || !is_object($doc->_attachments)) {
+ $doc->_attachments = new stdClass();
+ }
+
+ if (empty($mime_type)) {
+ $mime_type = $this->rest_client->content_mime_type($content);
+ }
+
+ $doc->_attachments->$name = new stdClass();
+ $doc->_attachments->$name->content_type = $mime_type;
+ $doc->_attachments->$name->data = base64_encode($content);
+ }
+
+ /**
+ * @param $doc
+ * @param $name
+ * @param $file
+ * Full path to a file (e.g. as returned by PHP's realpath function).
+ * @param $mime_type
+ * Optional. Will be auto-detected if not provided
+ * @return void
+ */
+ public function add_attachment_file($doc, $name, $file, $mime_type = null) {
+ $content = file_get_contents($file);
+ $this->add_attachment($doc, $name, $content, $mime_type);
+ }
+
+ /**
+ *
+ * Retrieve a document from CouchDB
+ *
+ * @throws SetteeWrongInputException
+ *
+ * @param $id
+ * Unique ID (usually: UUID) of the document to be retrieved.
+ * @return
+ * database document in PHP object format.
+ */
+ function get($id) {
+ if (empty($id)) {
+ throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
+ }
+
+ $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
+ $full_uri = str_replace("%3Frev%3D", "?rev=", $full_uri);
+ $ret = $this->rest_client->http_get($full_uri);
+ return $ret['decoded'];
+ }
+
+ /**
+ *
+ * Get the latest revision of a document with document id: $id in CouchDB.
+ *
+ * @throws SetteeWrongInputException
+ *
+ * @param $id
+ * Unique ID (usually: UUID) of the document to be retrieved.
+ * @return
+ * database document in PHP object format.
+ */
+ function get_rev($id) {
+ if (empty($id)) {
+ throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");
+ }
+
+ $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
+ $headers = $this->rest_client->http_head($full_uri);
+ if (empty($headers['Etag'])) {
+ throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");
+ }
+ $etag = str_replace('"', '', $headers['Etag']);
+ return $etag;
+ }
+
+ /**
+ * Delete a document
+ *
+ * @param $document
+ * a PHP object or JSON representation of the document that has _id and _rev fields.
+ *
+ * @return void
+ */
+ function delete($document) {
+ if (!is_object($document)) {
+ $document = json_decode($document);
+ }
+
+ $full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;
+ $this->rest_client->http_delete($full_uri);
+ }
+
+ /* ----------------- View-related functions -------------- */
+
+ /**
+ * Create a new view or update an existing one.
+ *
+ * @param $design_doc
+ * @param $view_name
+ * @param $map_src
+ * Source code of the map function in Javascript
+ * @param $reduce_src
+ * Source code of the reduce function in Javascript (optional)
+ * @return void
+ */
+ function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {
+ $obj = new stdClass();
+ $obj->_id = "_design/" . urlencode($design_doc);
+ $view_name = urlencode($view_name);
+ $obj->views->$view_name->map = $map_src;
+ if (!empty($reduce_src)) {
+ $obj->views->$view_name->reduce = $reduce_src;
+ }
+
+ // allow safe updates (even if slightly slower due to extra: rev-detection check).
+ return $this->save($obj, true);
+ }
+
+ /**
+ * Create a new view or update an existing one.
+ *
+ * @param $design_doc
+ * @param $view_name
+ * @param $key
+ * key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes
+ * that first element is startkey, second: endkey.
+ * @param $descending
+ * return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change
+ * order you also need to swap startkey and endkey values!
+ *
+ * @return void
+ */
+ function get_view($design_doc, $view_name, $key = null, $descending = false, $limit = false, $reduce = null, $startdocid = null) {
+ $id = "_design/" . urlencode($design_doc);
+ $view_name = urlencode($view_name);
+ $id .= "/_view/$view_name";
+
+ $data = array();
+ if (!empty($key)) {
+ if (is_string($key)) {
+ $data = "key=" . '"' . $key . '"';
+ } elseif (is_array($key)) {
+ list($startkey, $endkey) = $key;
+ $data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';
+ }
+
+ if ($descending) {
+ $data .= "&descending=true";
+ }
+ if ($startdocid != null) {
+ $data .= "&startkey_docid='$startdocid'";
+ }
+ if ($reduce === true) {
+ $data .= "&reduce=true";
+ } else if ($reduce === false){
+
+ $data .= "&reduce=false";
+ }
+ if ($limit) {
+ $data .= "&limit=" . $limit;
+ }
+ }
+
+
+
+ if (empty($id)) {
+ throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
+ }
+
+ $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
+
+ $full_uri = str_replace("%253Fgroup%253D", "?group=", $full_uri);
+ $full_uri = str_replace("%253Flimit%253D", "?limit=", $full_uri);
+ $ret = $this->rest_client->http_get($full_uri, $data);
+ //$ret['decoded'] = str_replace("?k","&k",$ret['decoded']);
+ return $ret['decoded'];
+ }
+
+ /**
+ * @param $id
+ * @return
+ * return a properly url-encoded id.
+ */
+ private function safe_urlencode($id) {
+ //-- System views like _design can have "/" in their URLs.
+ $id = rawurlencode($id);
+ if (substr($id, 0, 1) == '_') {
+ $id = str_replace('%2F', '/', $id);
+ }
+ return $id;
+ }
+
+ /** Getter for a database name */
+ function get_name() {
+ return $this->dbname;
+ }
}
+
--- a/couchdb/settee/src/classes/SetteeRestClient.class.php
+++ b/couchdb/settee/src/classes/SetteeRestClient.class.php
@@ -244,3 +244,4 @@
}
class SetteeRestClientException extends Exception {}
+
--- a/documents/about.php
+++ b/documents/about.php
@@ -1,7 +1,7 @@
<?php
include('template.inc.php');
-include_header_documents("");
+include_header_documents("About");
include_once('../include/common.inc.php');
?>
<h1>About</h1>
--- /dev/null
+++ b/documents/agency.php
@@ -1,1 +1,41 @@
+<?php
+include('template.inc.php');
+include_once('../include/common.inc.php');
+$agenciesdb = $server->get_db('disclosr-agencies');
+$idtoname = Array();
+foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
+ $idtoname[$row->id] = trim($row->value->name);
+}
+$foidocsdb = $server->get_db('disclosr-foidocuments');
+
+include_header_documents((isset($_REQUEST['id']) ? $idtoname[$_REQUEST['id']] : 'Entries by Agency'));
+$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
+?>
+<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
+<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
+<?php
+try {
+ if ($_REQUEST['id']) {
+ $rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows;
+ foreach ($rows as $row) {
+ //print_r($rows);
+ echo displayLogEntry($row, $idtoname);
+ if (!isset($startkey))
+ $startkey = $row->key;
+ $endkey = $row->key;
+ }
+ } else {
+ $rows = $foidocsdb->get_view("app", "byAgencyID?group=true", null, false, false, true)->rows;
+ if ($rows) {
+ foreach ($rows as $row) {
+ echo '<a href="agency.php?id=' . $row->key . '">' . $idtoname[$row->key] . " (" . $row->value . " records)</a> <br>\n";
+ }
+ }
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
+include_footer_documents();
+?>
--- a/documents/charts.php
+++ b/documents/charts.php
@@ -1,6 +1,6 @@
<?php
include('template.inc.php');
-include_header_documents("");
+include_header_documents("Charts");
include_once('../include/common.inc.php');
$agenciesdb = $server->get_db('disclosr-agencies');
@@ -15,29 +15,28 @@
<h1><a href="about.php">Charts</a></h1>
<h4 class="subheader">Lorem ipsum.</h4>
</div>
-<div id="employees" style="width:1000px;height:900px;"></div>
+<div id="bydate" style="width:1000px;height:300px;"></div>
+<div id="byagency" style="width:1200px;height:300px;"></div>
<script id="source">
window.onload = function() {
$(document).ready(function() {
var
d1 = [],
- start = new Date("2009/01/01 01:00").getTime(),
- options,
- graph,
- i, x, o;
+ options1,
+ o1;
<?php
try {
- $rows = $foidocsdb->get_view("app", "byDate?group=true", null, true)->rows;
+ $rows = $foidocsdb->get_view("app", "byDateMonthYear?group=true",null, false,false,true)->rows;
$dataValues = Array();
foreach ($rows as $row) {
- $dataValues[$row->value] = $row->key;
+ $dataValues[$row->key] = $row->value;
}
$i = 0;
ksort($dataValues);
- foreach ($dataValues as $value => $key) {
+ foreach ($dataValues as $key => $value) {
$date = date_create_from_format('Y-m-d', $key);
if (date_format($date, 'U') != "") {
echo " d1.push([".date_format($date, 'U')."000, $value]);" . PHP_EOL;
@@ -52,7 +51,7 @@
- options = {
+ options1 = {
xaxis : {
mode : 'time',
labelsAngle : 45
@@ -68,19 +67,19 @@
function drawGraph (opts) {
// Clone the options, so the 'options' variable always keeps intact.
- o = Flotr._.extend(Flotr._.clone(options), opts || {});
+ o1 = Flotr._.extend(Flotr._.clone(options1), opts || {});
// Return a new graph.
return Flotr.draw(
- document.getElementById("employees"),
+ document.getElementById("bydate"),
[ d1 ],
- o
+ o1
);
}
graph = drawGraph();
- Flotr.EventAdapter.observe(container, 'flotr:select', function(area){
+ Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:select', function(area){
// Draw selected area
graph = drawGraph({
xaxis : { min : area.x1, max : area.x2, mode : 'time', labelsAngle : 45 },
@@ -89,10 +88,74 @@
});
// When graph is clicked, draw the graph with default area.
- Flotr.EventAdapter.observe(container, 'flotr:click', function () { graph = drawGraph(); });
+ Flotr.EventAdapter.observe(document.getElementById("bydate"), 'flotr:click', function () { graph = drawGraph(); });
});
};
+
+var d2 = [];
+var agencylabels = [];
+function agencytrackformatter(obj) {
+
+ return agencylabels[Math.floor(obj.x)] +" = "+obj.y;
+
+ }
+ function agencytickformatter(val, axis) {
+ if (agencylabels[Math.floor(val)]) {
+ return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(agencylabels[Math.floor(val)])+"</b>";
+
+ } else {
+ return "";
+ }
+ }
+<?php
+ try {
+ $rows = $foidocsdb->get_view("app", "byAgencyID?group=true",null, false,false,true)->rows;
+
+
+ $dataValues = Array();
+ $i = 0;
+ foreach ($rows as $row) {
+ echo " d2.push([".$i.", $row->value]);" . PHP_EOL;
+ echo " agencylabels.push(['".str_replace("'","",$idtoname[$row->key])."']);" . PHP_EOL;
+
+ $i++;
+ }
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ ?>
+ // Draw the graph
+ Flotr.draw(
+ document.getElementById("byagency"),
+ [d2],
+ {
+ bars : {
+ show : true,
+ horizontal : false,
+ shadowSize : 0,
+ barWidth : 0.5
+ },
+mouse : {
+ track : true,
+ relative : true,
+ trackFormatter: agencytrackformatter
+ },
+ yaxis : {
+ min : 0,
+ autoscaleMargin : 1
+ },
+ xaxis: {
+ minorTickFreq: 1,
+ noTicks: agencylabels.length,
+ showMinorLabels: true,
+ tickFormatter: agencytickformatter
+ },
+ legend: {
+ show: false
+ }
+ }
+ );
</script>
<?php
--- /dev/null
+++ b/documents/date.php
@@ -1,1 +1,34 @@
+<?php
+include('template.inc.php');
+include_header_documents("Entries by Date");
+include_once('../include/common.inc.php');
+$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
+?>
+<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
+<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
+<?php
+/*$agenciesdb = $server->get_db('disclosr-agencies');
+
+$idtoname = Array();
+foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
+ $idtoname[$row->id] = trim($row->value->name);
+}
+$foidocsdb = $server->get_db('disclosr-foidocuments');
+try {
+ $rows = $foidocsdb->get_view("app", "byDate", Array($endkey, '0000-00-00'), true, 20)->rows;
+ if ($rows) {
+ foreach ($rows as $key => $row) {
+ echo displayLogEntry($row, $idtoname);
+ if (!isset($startkey)) $startkey = $row->key;
+ $endkey = $row->key;
+ }
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
+*/
+include_footer_documents();
+?>
+
--- a/documents/disclogsList.php
+++ b/documents/disclogsList.php
@@ -1,7 +1,7 @@
<?php
include('template.inc.php');
-include_header_documents("");
+include_header_documents("List of Disclosure Logs");
include_once('../include/common.inc.php');
echo "<table>
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -13,6 +13,8 @@
from datetime import *
import codecs
+import difflib
+
from StringIO import StringIO
from pdfminer.pdfparser import PDFDocument, PDFParser
@@ -49,6 +51,31 @@
""" do the scraping """
return
+class GenericHTMLDisclogScraper(GenericDisclogScraper):
+
+ def doScrape(self):
+ foidocsdb = scrape.couch['disclosr-foidocuments']
+ (url, mime_type, rcontent) = scrape.fetchURL(scrape.docsdb,
+ self.getURL(), "foidocuments", self.getAgencyID())
+ content = rcontent
+ dochash = scrape.mkhash(content)
+ doc = foidocsdb.get(dochash)
+ if doc is None:
+ print "saving " + dochash
+ description = "This log may have updated but as it was not in a table last time we viewed it, we cannot extract what has changed. Please refer to the agency's website Disclosure Log to see the most recent entries"
+ last_attach = scrape.getLastAttachment(scrape.docsdb, self.getURL())
+ if last_attach != None:
+ html_diff = difflib.HtmlDiff()
+ description = description + "\nChanges: "
+ description = description + html_diff.make_table(last_attach.read().split('\n'),
+ content.split('\n'))
+ edate = date.today().strftime("%Y-%m-%d")
+ doc = {'_id': dochash, 'agencyID': self.getAgencyID()
+ , 'url': self.getURL(), 'docID': dochash,
+ "date": edate, "title": "Disclosure Log Updated", "description": description}
+ foidocsdb.save(doc)
+ else:
+ print "already saved"
class GenericPDFDisclogScraper(GenericDisclogScraper):
@@ -62,7 +89,7 @@
device = TextConverter(rsrcmgr, outfp, codec='utf-8',
laparams=laparams)
fp = StringIO()
- fp.write(content.read())
+ fp.write(content)
process_pdf(rsrcmgr, device, fp, set(), caching=True,
check_extractable=True)
Binary files /dev/null and b/documents/img/feed-icon-14x14.png differ
--- a/documents/index.php
+++ b/documents/index.php
@@ -1,12 +1,13 @@
<?php
-
include('template.inc.php');
include_header_documents("");
include_once('../include/common.inc.php');
-$startkey = (isset($_REQUEST['start_key']) ? $_REQUEST['start_key'] : '9999-99-99');
+$endkey = (isset($_REQUEST['end_key']) ? $_REQUEST['end_key'] : '9999-99-99');
+$enddocid = (isset($_REQUEST['end_docid']) ? $_REQUEST['end_docid'] : null);
?>
+<div class="headline">Read all the information released by Australian Federal Government agencies under the FOI Act in one place!</div>
+<a style='float:right' href="rss.xml.php"><img src="img/feed-icon-14x14.png" alt="RSS Icon"/> All Agencies RSS Feed</a><br>
<?php
-
$agenciesdb = $server->get_db('disclosr-agencies');
$idtoname = Array();
@@ -15,17 +16,20 @@
}
$foidocsdb = $server->get_db('disclosr-foidocuments');
try {
- $rows = $foidocsdb->get_view("app", "byDate", Array($startkey, '0000-00-00'), true, 20)->rows;
+ $rows = $foidocsdb->get_view("app", "byDate", Array($endkey, '0000-00-00'), true, 20,null, $enddocid)->rows;
if ($rows) {
foreach ($rows as $key => $row) {
echo displayLogEntry($row, $idtoname);
+ if (!isset($startkey))
+ $startkey = $row->key;
$endkey = $row->key;
+ $enddocid = $row->value->_id;
}
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
}
-echo "<a href='?start_key=$endkey'>next page</a>";
+echo "<a class='btn btn-large btn-primary' href='?end_key=$endkey&end_docid=$enddocid' style='float:right;'>next page <i class='icon-circle-arrow-right icon-white'></i></a>";
include_footer_documents();
?>
--- a/documents/rss.xml.php
+++ b/documents/rss.xml.php
@@ -8,21 +8,28 @@
//Creating an instance of FeedWriter class.
$TestFeed = new RSS2FeedWriter();
//Setting the channel elements
-//Use wrapper functions for common channelelements
-$TestFeed->setTitle('disclosurelo.gs Newest Entries - All');
-$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php');
-$TestFeed->setDescription('disclosurelo.gs Newest Entries - All Agencies');
-$TestFeed->setChannelElement('language', 'en-us');
-$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));
-
-//Retriving informations from database
+////Retriving informations from database
$idtoname = Array();
$agenciesdb = $server->get_db('disclosr-agencies');
foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) {
$idtoname[$row->id] = trim($row->value->name);
}
$foidocsdb = $server->get_db('disclosr-foidocuments');
-$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99', '0000-00-00', 50), true)->rows;
+if (isset($_REQUEST['id'])) {
+ $rows = $foidocsdb->get_view("app", "byAgencyID", $_REQUEST['id'], false, false, false)->rows;
+ $title = $idtoname[$_REQUEST['id']];
+} else {
+ $rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99', '0000-00-00', 50), true)->rows;
+ $title = 'All Agencies';
+}
+//Use wrapper functions for common channelelements
+$TestFeed->setTitle('disclosurelo.gs Newest Entries - '.$title);
+$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php'.(isset($_REQUEST['id'])? '?id='.$_REQUEST['id'] : ''));
+$TestFeed->setDescription('disclosurelo.gs Newest Entries - '.$title);
+$TestFeed->setChannelElement('language', 'en-us');
+$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));
+
+
//print_r($rows);
foreach ($rows as $row) {
//Create an empty FeedItem
--- a/documents/runScrapers.sh
+++ b/documents/runScrapers.sh
@@ -1,3 +1,10 @@
-for f in scrapers/*.py; do echo "Processing $f file.."; python $f; done
+for f in scrapers/*.py;
+ do echo "Processing $f file..";
+ python $f;
+ if [ "$?" -ne "0" ]; then
+ echo "error";
+ sleep 2;
+ fi
+done
--- a/documents/scrape.py
+++ b/documents/scrape.py
@@ -76,6 +76,16 @@
addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
addinfourl.code = code
return addinfourl
+
+def getLastAttachment(docsdb,url):
+ hash = mkhash(url)
+ doc = docsdb.get(hash)
+ if doc != None:
+ last_attachment_fname = doc["_attachments"].keys()[-1]
+ last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
+ return last_attachment
+ else:
+ return None
def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True):
url = canonurl(url)
@@ -94,10 +104,10 @@
last_attachment_fname = doc["_attachments"].keys()[-1]
last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
content = last_attachment
- return (doc['url'],doc['mime_type'],content)
+ return (doc['url'],doc['mime_type'],content.read())
if scrape_again == False:
print "Not scraping this URL again as requested"
- return (None,None,None)
+ return (doc['url'],doc['mime_type'],content.read())
req.add_header("User-Agent", "Mozilla/4.0 (compatible; Prometheus webspider; owner maxious@lambdacomplex.org)")
#if there is a previous version stored in couchdb, load caching helper tags
@@ -131,7 +141,7 @@
last_attachment_fname = doc["_attachments"].keys()[-1]
last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
content = last_attachment
- return (doc['url'],doc['mime_type'],content)
+ return (doc['url'],doc['mime_type'],content.read())
else:
print "new webpage loaded"
content = url_handle.read()
--- a/documents/scrapers/0049d35216493c545ef5f7f000e6b252.py
+++ b/documents/scrapers/0049d35216493c545ef5f7f000e6b252.py
@@ -2,7 +2,14 @@
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers
-
+import traceback
+try:
+ import amonpy
+ amonpy.config.address = 'http://amon_instance:port'
+ amonpy.config.secret_key = 'the secret key from /etc/amon.conf'
+ amon_available = True
+except ImportError:
+ amon_available = False
class ScraperImplementation(genericScrapers.GenericPDFDisclogScraper):
@@ -12,8 +19,30 @@
if __name__ == '__main__':
print 'Subclass:', issubclass(ScraperImplementation,
- genericScrapers.GenericOAICDisclogScraper)
+ genericScrapers.GenericPDFDisclogScraper)
print 'Instance:', isinstance(ScraperImplementation(),
- genericScrapers.GenericOAICDisclogScraper)
- ScraperImplementation().doScrape()
+ genericScrapers.GenericPDFDisclogScraper)
+ try:
+ ScraperImplementation().doScrape()
+ except Exception, err:
+ sys.stderr.write('ERROR: %s\n' % str(err))
+ print ‘Error Reason: ‘, err.__doc__
+ print ‘Exception: ‘, err.__class__
+ print traceback.format_exc()
+ if amon_available:
+ data = {
+ 'exception_class': '',
+ 'url': '',
+ 'backtrace': ['exception line ', 'another exception line'],
+ 'enviroment': '',
+
+ # In 'data' you can add request information, session variables - it's a recursive
+ # dictionary, so you can literally add everything important for your specific case
+ # The dictionary doesn't have a specified structure, the keys below are only example
+ 'data': {'request': '', 'session': '', 'more': ''}
+ }
+
+ amonpy.exception(data)
+ pass
+
--- /dev/null
+++ b/documents/scrapers/0ae822d1a748e60d90f0b79b97d5a3e5.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- /dev/null
+++ b/documents/scrapers/31685505438d393f45a90f442b8fa27f.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericPDFDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericPDFDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericPDFDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/31685505438d393f45a90f442b8fa27f.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-pdf
--- /dev/null
+++ b/documents/scrapers/3e2f110af49d62833a835bd257771ffb.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/3e2f110af49d62833a835bd257771ffb.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- /dev/null
+++ b/documents/scrapers/41a166419503bb50e410c58be54c102f.py
@@ -1,1 +1,27 @@
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+import scrape
+from bs4 import BeautifulSoup
+from datetime import date
+#http://www.doughellmann.com/PyMOTW/abc/
+class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
+ def getTable(self,soup):
+ return soup.find(id= "ctl00_MSO_ContentDiv").table
+
+ def getColumns(self,columns):
+ (id, title, description, notes) = columns
+ return (id, title, title, description, notes)
+ def getDate(self, content, entry, doc):
+ edate = date.today().strftime("%Y-%m-%d")
+ doc.update({'date': edate})
+ return
+ def getColumnCount(self):
+ return 4
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/41a166419503bb50e410c58be54c102f.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-aspx
+
--- /dev/null
+++ b/documents/scrapers/4d2af2dcc72f1703bbf04b13b03720a8.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/4d2af2dcc72f1703bbf04b13b03720a8.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- a/documents/scrapers/50601505ef69483121a6d130bb0515e4.txt
+++ b/documents/scrapers/50601505ef69483121a6d130bb0515e4.txt
@@ -1,1 +1,1 @@
-apsc has ACMA style disclog
+ACMA style
--- /dev/null
+++ b/documents/scrapers/525c3953187da08cd702359b2fc2997f.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/525c3953187da08cd702359b2fc2997f.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- /dev/null
+++ b/documents/scrapers/627f116dfe42c9f27ad6747be0aa44e2.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/627f116dfe42c9f27ad6747be0aa44e2.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- /dev/null
+++ b/documents/scrapers/795e7a8afb39a420360aa207b0cb1306.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/795e7a8afb39a420360aa207b0cb1306.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- /dev/null
+++ b/documents/scrapers/7b39ce7f362a0af9a711eaf223943eea.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/7b39ce7f362a0af9a711eaf223943eea.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- /dev/null
+++ b/documents/scrapers/7c6adc1d41cf029bf1a0959e5156477a.py
@@ -1,1 +1,51 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+import scrape
+from datetime import date
+from pyquery import PyQuery as pq
+from lxml import etree
+import urllib
+import dateutil
+from dateutil.parser import *
+class ACMADisclogScraper(genericScrapers.GenericDisclogScraper):
+
+ def doScrape(self):
+ foidocsdb = scrape.couch['disclosr-foidocuments']
+ (url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
+ self.getURL(), "foidocuments", self.getAgencyID())
+
+ d = pq(content)
+ d.make_links_absolute(base_url = self.getURL())
+ for table in d('table').items():
+ title= table('thead').text()
+ print title
+ (idate,descA,descB,link,deldate,notes) = table('tbody tr').map(lambda i, e: pq(e).children().eq(1).text())
+ links = table('a').map(lambda i, e: pq(e).attr('href'))
+ description = descA+" "+descB
+ edate = parse(idate[:12], dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
+ print edate
+ dochash = scrape.mkhash(self.remove_control_chars(title))
+ doc = foidocsdb.get(dochash)
+ if doc is None:
+ print "saving " + dochash
+ edate = date.today().strftime("%Y-%m-%d")
+ doc = {'_id': dochash, 'agencyID': self.getAgencyID()
+ , 'url': self.getURL(), 'docID': dochash,
+ "links": links,
+ "date": edate, "notes": notes, "title": title, "description": description}
+ #print doc
+ foidocsdb.save(doc)
+ else:
+ print "already saved"
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ACMADisclogScraper,
+ genericScrapers.GenericDisclogScraper)
+ print 'Instance:', isinstance(ACMADisclogScraper(),
+ genericScrapers.GenericDisclogScraper)
+ ACMADisclogScraper().doScrape()
+
--- a/documents/scrapers/7c6adc1d41cf029bf1a0959e5156477a.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-acma style
+
--- /dev/null
+++ b/documents/scrapers/9f4815bfdcb918a036e4bb43a30f8d77.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/9f4815bfdcb918a036e4bb43a30f8d77.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- /dev/null
+++ b/documents/scrapers/bb96fe4065afb7e0872136dd657f9369.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/bb96fe4065afb7e0872136dd657f9369.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- /dev/null
+++ b/documents/scrapers/bf6e587f166040b63681cd2ff76fbfdf.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/bf6e587f166040b63681cd2ff76fbfdf.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- a/documents/scrapers/c1302c8d7cbbd911f0d4d8a4128f8079.txt
+++ b/documents/scrapers/c1302c8d7cbbd911f0d4d8a4128f8079.txt
@@ -1,1 +1,1 @@
-uses RET disclog
+parent
--- /dev/null
+++ b/documents/scrapers/cb7f40e3495b682de6eee61bf09c1cfc.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/cb7f40e3495b682de6eee61bf09c1cfc.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- /dev/null
+++ b/documents/scrapers/d72744fb1e5d6e87af9a5ea16cc27fa5.py
@@ -1,1 +1,49 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+import scrape
+from datetime import date
+from pyquery import PyQuery as pq
+from lxml import etree
+import urllib
+import dateutil
+from dateutil.parser import *
+class ACMADisclogScraper(genericScrapers.GenericDisclogScraper):
+
+ def doScrape(self):
+ foidocsdb = scrape.couch['disclosr-foidocuments']
+ (url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
+ self.getURL(), "foidocuments", self.getAgencyID())
+
+ d = pq(content)
+ d.make_links_absolute(base_url = self.getURL())
+ for item in d('.item-list').items():
+ title= item('h3').text()
+ print title
+ links = item('a').map(lambda i, e: pq(e).attr('href'))
+ description = title= item('ul').text()
+ edate = date.today().strftime("%Y-%m-%d")
+ print edate
+ dochash = scrape.mkhash(self.remove_control_chars(title))
+ doc = foidocsdb.get(dochash)
+ if doc is None:
+ print "saving " + dochash
+ doc = {'_id': dochash, 'agencyID': self.getAgencyID()
+ , 'url': self.getURL(), 'docID': dochash,
+ "links": links,
+ "date": edate, "title": title, "description": description}
+ #print doc
+ foidocsdb.save(doc)
+ else:
+ print "already saved"
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ACMADisclogScraper,
+ genericScrapers.GenericDisclogScraper)
+ print 'Instance:', isinstance(ACMADisclogScraper(),
+ genericScrapers.GenericDisclogScraper)
+ ACMADisclogScraper().doScrape()
+
--- a/documents/scrapers/d72744fb1e5d6e87af9a5ea16cc27fa5.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-acma style
+
--- /dev/null
+++ b/documents/scrapers/e770921522a49dc77de208cc724ce134.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/e770921522a49dc77de208cc724ce134.txt
+++ /dev/null
@@ -1,1 +1,1 @@
-no disclog
+
--- /dev/null
+++ b/documents/scrapers/f189459fc43f941e0d4ecfba52c666f3.py
@@ -1,1 +1,19 @@
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+
+class ScraperImplementation(genericScrapers.GenericHTMLDisclogScraper):
+
+ def __init__(self):
+ super(ScraperImplementation, self).__init__()
+
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(ScraperImplementation,
+ genericScrapers.GenericHTMLDisclogScraper)
+ print 'Instance:', isinstance(ScraperImplementation(),
+ genericScrapers.GenericHTMLDisclogScraper)
+ ScraperImplementation().doScrape()
+
--- a/documents/scrapers/f189459fc43f941e0d4ecfba52c666f3.txt
+++ /dev/null
@@ -1,2 +1,1 @@
-no disclog
--- a/documents/sitemap.xml.php
+++ b/documents/sitemap.xml.php
@@ -10,10 +10,18 @@
if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php")
echo " <url><loc>" . local_url() . "$file</loc><priority>0.6</priority></url>\n";
}
-
-$db = $server->get_db('disclosr-foidocuments');
+$agenciesdb = $server->get_db('disclosr-agencies');
try {
- $rows = $db->get_view("app", "all")->rows;
+ $rows = $agenciesdb->get_view("app", "byCanonicalName")->rows;
+ foreach ($rows as $row) {
+ echo '<url><loc>' . local_url() . 'agency.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+$foidocsdb = $server->get_db('disclosr-foidocuments');
+try {
+ $rows = $foidocsdb->get_view("app", "all")->rows;
foreach ($rows as $row) {
echo '<url><loc>' . local_url() . 'view.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
}
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -61,14 +61,17 @@
<a class="brand" href="#">Australian Disclosure Logs</a>
<div class="nav-collapse collapse">
<p class="navbar-text pull-right">
- Check out our subsites on:
+ <small>
+ Subsites on:
+ </small>
<a href="http://orgs.disclosurelo.gs">Government Agencies</a>
• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>
• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>
</p>
<ul class="nav">
- <li><a href="index.php">Home</a></li>
+ <li><a href="agency.php">By Agency</a></li>
+ <li><a href="date.php">By Date</a></li>
<li><a href="disclogsList.php">List of Disclosure Logs</a></li>
<li><a href="about.php">About</a></li>
@@ -82,6 +85,7 @@
}
function include_footer_documents() {
+ global $ENV;
?>
</div> <!-- /container -->
<hr>
@@ -89,7 +93,9 @@
<footer>
<p>Not affiliated with or endorsed by any government agency.</p>
</footer>
- <script type="text/javascript">
+ <?php
+ if ($ENV != "DEV") {
+ echo "<script type='text/javascript'>
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-12341040-4']);
@@ -106,7 +112,9 @@
s.parentNode.insertBefore(ga, s);
})();
- </script>
+ </script>";
+ }
+ ?>
<!-- Le javascript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
@@ -149,8 +157,8 @@
function displayLogEntry($row, $idtoname) {
$result = "";
$result .= '<div itemscope itemtype="http://schema.org/Article">';
- $result .= '<h2> <span itemprop="datePublished">' . $row->value->date . "</span>: <span itemprop='name headline'>" . truncate($row->value->title, 120) . "</span>";
- $result .= ' (<span itemprop="author publisher creator">' . $idtoname[$row->value->agencyID] . '</span>)</h2>';
+ $result .= '<h2><a href="http://disclosurelo.gs/view.php?id='.$row->value->_id.'"> <span itemprop="datePublished">' . $row->value->date . "</span>: <span itemprop='name headline'>" . truncate($row->value->title, 120) . "</span>";
+ $result .= ' (<span itemprop="author publisher creator">' . $idtoname[$row->value->agencyID] . '</span>)</a></h2>';
$result .= "<p itemprop='description articleBody text'> Title: " . $row->value->title . "<br/>";
if (isset($row->value->description)) {
$result .= str_replace("\n", "<br>", preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "",trim($row->value->description)));
@@ -163,13 +171,13 @@
if (isset($row->value->links)) {
$result .= '<h3>Links/Documents</h3><ul itemprop="associatedMedia">';
foreach ($row->value->links as $link) {
- $result .= '<li itemscope itemtype="http://schema.org/MediaObject"><a href=' . $link . ' itemprop="url contentURL">' . urlencode($link) . "</a></li>";
+ $result .= '<li itemscope itemtype="http://schema.org/MediaObject"><a href="' . htmlspecialchars ($link) . '" itemprop="url contentURL">' . htmlspecialchars ( $link) . "</a></li>";
}
$result .= "</ul>";
}
$result .= "<small><A itemprop='url' href='" . $row->value->url . "'>View original source...</a> ID: " . strip_tags($row->value->docID) . "</small>";
- $result .= "</div>";
+ $result .= "</div>\n";
return $result;
}
--- a/documents/view.php
+++ b/documents/view.php
@@ -1,6 +1,5 @@
<?php
include('template.inc.php');
-include_header_documents("");
include_once('../include/common.inc.php');
?>
<?php
@@ -17,6 +16,8 @@
try {
$obj = new stdClass();
$obj->value = $foidocsdb->get($_REQUEST['id']);
+ include_header_documents($obj->value->title);
+
echo displayLogEntry($obj,$idtoname);
} catch (SetteeRestClientException $e) {
--- a/getAgency.php
+++ b/getAgency.php
@@ -5,11 +5,11 @@
function displayValue($key, $value, $mode) {
global $db, $schemas;
if ($mode == "view") {
- if (strpos($key, "_") === 0 || $key == "metadata")
+ if (strpos($key, "_") === 0 || $key == "metadata" || $key == "metaTags" || $key == "statistics")
return;
echo "<tr>";
- echo "<td>";
+ echo "<td class='$key'>";
if (isset($schemas['agency']["properties"][$key])) {
echo $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small>";
}
--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -9,6 +9,7 @@
|| strstr($_SERVER['PHP_SELF'], "include/")
|| strstr($_SERVER['PHP_SELF'], "documents/")
|| $_SERVER['SERVER_NAME'] == "disclosurelo.gs"
+ || $_SERVER['SERVER_NAME'] == "www.disclosurelo.gs"
)
$basePath = "../";
