pagination
Former-commit-id: 65ec7105144f184adf61844dc454c637d8b350ea
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,6 @@
[submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git
-[submodule "couchdb/settee"]
- path = couchdb/settee
- url = https://github.com/inadarei/settee.git
[submodule "lib/php-diff"]
path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git
--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -4,74 +4,62 @@
//function createFOIDocumentsDesignDoc() {
$foidb = $server->get_db('disclosr-foidocuments');
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
- $obj->views->byDate->reduce = "_count";
- $obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
- $obj->views->byAgencyID->reduce = "_count";
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
+$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
+$obj->views->byDate->reduce = "_count";
+$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";
+$obj->views->byDateMonthYear->reduce = "_count";
+$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
+$obj->views->byAgencyID->reduce = "_count";
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- $foidb->save($obj, true);
+// allow safe updates (even if slightly slower due to extra: rev-detection check).
+$foidb->save($obj, true);
-function createDocumentsDesignDoc() {
- /*
- global $db;
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
- "views": {
- "web_server": {
- "map": "function(doc) {\n emit(doc.web_server, 1);\n}",
- "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
- },
- "byAgency": {
- "map": "function(doc) {\n emit(doc.agencyID, 1);\n}",
- "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
- },
- "byURL": {
- "map": "function(doc) {\n emit(doc.url, doc);\n}"
- },
- "agency": {
- "map": "function(doc) {\n emit(doc.agencyID, doc);\n}"
- },
- "byWebServer": {
- "map": "function(doc) {\n emit(doc.web_server, doc);\n}"
- },
- "getValidationRequired": {
- "map": "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}"
- }
- } */
-}
+//function createDocumentsDesignDoc() {
+$docdb = $server->get_db('disclosr-documents');
+
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->web_server->map = "function(doc) {\n emit(doc.web_server, 1);\n}";
+$obj->views->web_server->reduce = "function (key, values, rereduce) {\n return sum(values);\n}";
+$obj->views->byAgency->map = "function(doc) {\n emit(doc.agencyID, 1);\n}";
+$obj->views->byAgency->reduce = "function (key, values, rereduce) {\n return sum(values);\n}";
+$obj->views->byURL->map = "function(doc) {\n emit(doc.url, doc);\n}";
+$obj->views->agency->map = "function(doc) {\n emit(doc.agencyID, doc);\n}";
+$obj->views->byWebServer->map = "function(doc) {\n emit(doc.web_server, doc);\n}";
+$obj->views->getValidationRequired = "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}";
+
+
+
//function createAgencyDesignDoc() {
$db = $server->get_db('disclosr-agencies');
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
- $obj->views->byCanonicalName->map = "function(doc) {
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
+$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
+$obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc);
}
};";
- $obj->views->byDeptStateName->map = "function(doc) {
+$obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id);
}
};";
- $obj->views->parentOrgs->map = "function(doc) {
+$obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) {
emit(doc._id, doc.parentOrg);
}
};";
- $obj->views->byName->map = 'function(doc) {
+$obj->views->byName->map = 'function(doc) {
if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
emit(doc.name, doc._id);
if (typeof(doc.shortName) != "undefined" && doc.shortName != doc.name) {
@@ -95,14 +83,14 @@
}
};';
- $obj->views->foiEmails->map = "function(doc) {
+$obj->views->foiEmails->map = "function(doc) {
emit(doc._id, doc.foiEmail);
};";
- $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
- $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
- $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
- $obj->views->getScrapeRequired->map = "function(doc) {
+$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
+$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
+$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
+$obj->views->getScrapeRequired->map = "function(doc) {
var lastScrape = Date.parse(doc.metadata.lastScraped);
@@ -113,14 +101,14 @@
}
};";
- $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
- $obj->views->getConflicts->map = "function(doc) {
+$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
+$obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts));
}
}";
- // http://stackoverflow.com/questions/646628/javascript-startswith
- $obj->views->score->map = 'if(!String.prototype.startsWith){
+// http://stackoverflow.com/questions/646628/javascript-startswith
+$obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {
return !this.indexOf(str);
}
@@ -144,7 +132,7 @@
emit(count+doc._id, {id:doc._id, name: doc.name, score:count, orgType: doc.orgType, portfolio:portfolio});
}
}';
- $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
+$obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {
return !this.indexOf(str);
}
@@ -164,22 +152,20 @@
emit("total", 1);
}
}';
- $obj->views->scoreHas->reduce = 'function (key, values, rereduce) {
+$obj->views->scoreHas->reduce = 'function (key, values, rereduce) {
return sum(values);
}';
- $obj->views->fieldNames->map = '
+$obj->views->fieldNames->map = '
function(doc) {
for(var propName in doc) {
emit(propName, doc._id);
}
}';
- $obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
+$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
return values.length;
}';
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- $db->save($obj, true);
-
-
+// allow safe updates (even if slightly slower due to extra: rev-detection check).
+$db->save($obj, true);
?>
--- a/couchdb/SetteeDatabase.class.php
+++ /dev/null
@@ -1,306 +1,1 @@
-<?php
-/**
-* Databaase class.
-*/
-class SetteeDatabase {
-
- /**
- * Base URL of the CouchDB REST API
- */
- private $conn_url;
-
- /**
- * HTTP REST Client instance
- */
- protected $rest_client;
-
- /**
- * Name of the database
- */
- private $dbname;
-
- /**
- * Default constructor
- */
- function __construct($conn_url, $dbname) {
- $this->conn_url = $conn_url;
- $this->dbname = $dbname;
- $this->rest_client = SetteeRestClient::get_instance($this->conn_url);
- }
-
-
- /**
- * Get UUID from CouchDB
- *
- * @return
- * CouchDB-generated UUID string
- *
- */
- function gen_uuid() {
- $ret = $this->rest_client->http_get('_uuids');
- return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking
- }
-
- /**
- * Create or update a document database
- *
- * @param $document
- * PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.
- *
- * <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).
- * If "_id" is missing, CouchDB will be used to generate a UUID.
- *
- * <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.
- * You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be
- * one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but
- * not provide "_id" since that is an invalid input.
- *
- * @param $allowRevAutoDetection
- * Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision
- * for a document and use it. This option is "false" by default because it involves an extra http HEAD request and
- * therefore can make save() operation slightly slower if such auto-detection is not required.
- *
- * @return
- * document object with the database id (uuid) and revision attached;
- *
- * @throws SetteeCreateDatabaseException
- */
- function save($document, $allowRevAutoDetection = false) {
- if (is_string($document)) {
- $document = json_decode($document);
- }
-
- // Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)
- if(is_array($document)) {
- $document = (object) $document;
- }
-
- if (empty($document->_id) && empty($document->_rev)) {
- $id = $this->gen_uuid();
- }
- elseif (empty($document->_id) && !empty($document->_rev)) {
- throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");
- }
- else {
- $id = $document->_id;
-
- if ($allowRevAutoDetection) {
- try {
- $rev = $this->get_rev($id);
- } catch (SetteeRestClientException $e) {
- // auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error
- }
- if (!empty($rev)) {
- $document->_rev = $rev;
- }
- }
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $document_json = json_encode($document, JSON_NUMERIC_CHECK);
-
- $ret = $this->rest_client->http_put($full_uri, $document_json);
-
- $document->_id = $ret['decoded']->id;
- $document->_rev = $ret['decoded']->rev;
-
- return $document;
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $content
- * Content of the attachment in a string-buffer format. This function will automatically base64-encode content for
- * you, so you don't have to do it.
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment($doc, $name, $content, $mime_type = null) {
- if (empty($doc->_attachments) || !is_object($doc->_attachments)) {
- $doc->_attachments = new stdClass();
- }
-
- if (empty($mime_type)) {
- $mime_type = $this->rest_client->content_mime_type($content);
- }
-
- $doc->_attachments->$name = new stdClass();
- $doc->_attachments->$name->content_type = $mime_type;
- $doc->_attachments->$name->data = base64_encode($content);
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $file
- * Full path to a file (e.g. as returned by PHP's realpath function).
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment_file($doc, $name, $file, $mime_type = null) {
- $content = file_get_contents($file);
- $this->add_attachment($doc, $name, $content, $mime_type);
- }
-
- /**
- *
- * Retrieve a document from CouchDB
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri);
- $ret = $this->rest_client->http_get($full_uri);
- return $ret['decoded'];
- }
-
- /**
- *
- * Get the latest revision of a document with document id: $id in CouchDB.
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get_rev($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $headers = $this->rest_client->http_head($full_uri);
- if (empty($headers['Etag'])) {
- throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");
- }
- $etag = str_replace('"', '', $headers['Etag']);
- return $etag;
- }
-
- /**
- * Delete a document
- *
- * @param $document
- * a PHP object or JSON representation of the document that has _id and _rev fields.
- *
- * @return void
- */
- function delete($document) {
- if (!is_object($document)) {
- $document = json_decode($document);
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;
- $this->rest_client->http_delete($full_uri);
- }
-
-
- /*----------------- View-related functions --------------*/
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $map_src
- * Source code of the map function in Javascript
- * @param $reduce_src
- * Source code of the reduce function in Javascript (optional)
- * @return void
- */
- function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $obj->views->$view_name->map = $map_src;
- if (!empty($reduce_src)) {
- $obj->views->$view_name->reduce = $reduce_src;
- }
-
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- return $this->save($obj, true);
- }
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $key
- * key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes
- * that first element is startkey, second: endkey.
- * @param $descending
- * return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change
- * order you also need to swap startkey and endkey values!
- *
- * @return void
- */
- function get_view($design_doc, $view_name, $key = null, $descending = false) {
- $id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $id .= "/_view/$view_name";
-
- $data = array();
- if (!empty($key)) {
- if (is_string($key)) {
- $data = "key=" . '"' . $key . '"';
- }
- elseif (is_array($key)) {
- list($startkey, $endkey) = $key;
- $data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';
- }
-
- if ($descending) {
- $data .= "&descending=true";
- }
- }
-
-
-
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%253Fgroup%253Dtrue","?group=true",$full_uri);
- $ret = $this->rest_client->http_get($full_uri, $data);
- return $ret['decoded'];
-
- }
-
- /**
- * @param $id
- * @return
- * return a properly url-encoded id.
- */
- private function safe_urlencode($id) {
- //-- System views like _design can have "/" in their URLs.
- $id = rawurlencode($id);
- if (substr($id, 0, 1) == '_') {
- $id = str_replace('%2F', '/', $id);
- }
- return $id;
- }
-
- /** Getter for a database name */
- function get_name() {
- return $this->dbname;
- }
-
-}
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -94,7 +94,7 @@
def getDate(self, content, entry, doc):
date = ''.join(content.stripped_strings).strip()
(a,b,c) = date.partition("(")
- date = a.replace("Octber","October")
+ date = self.remove_control_chars(a.replace("Octber","October"))
print date
edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
print edate
@@ -121,7 +121,7 @@
columns = row.find_all('td')
if len(columns) == self.getColumnCount():
(id, date, title, description, notes) = self.getColumns(columns)
- print ''.join(id.stripped_strings).encode('ascii', 'ignore')
+ print self.remove_control_chars(''.join(id.stripped_strings))
if id.string == None:
hash = scrape.mkhash(self.remove_control_chars(url+(''.join(date.stripped_strings))))
else:
--- a/documents/index.php
+++ b/documents/index.php
@@ -1,11 +1,11 @@
<?php
+
include('template.inc.php');
include_header_documents("");
include_once('../include/common.inc.php');
+$startkey = (isset($_REQUEST['start_key']) ? $_REQUEST['start_key'] : '9999-99-99');
?>
<?php
-
-
$agenciesdb = $server->get_db('disclosr-agencies');
@@ -15,17 +15,17 @@
}
$foidocsdb = $server->get_db('disclosr-foidocuments');
try {
- $rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00'), true)->rows;
-
-
+ $rows = $foidocsdb->get_view("app", "byDate", Array($startkey, '0000-00-00'), true, 20)->rows;
if ($rows) {
- foreach ($rows as $row) {
-echo displayLogEntry($row,$idtoname);
+ foreach ($rows as $key => $row) {
+ echo displayLogEntry($row, $idtoname);
+ $endkey = $row->key;
}
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
}
+echo "<a href='?start_key=$endkey'>next page</a>";
include_footer_documents();
?>
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -145,7 +145,7 @@
$result .= "</ul>";
}
- $result .= "<small><A href='".$row->value->url."'>View original source...</a> ID: ".$row->value->docID."</small>";
+ $result .= "<small><A href='".$row->value->url."'>View original source...</a> ID: ".strip_tags($row->value->docID)."</small>";
$result .= "</div>";
return $result;
}