rm settee
Former-commit-id: 2fe3d86753e524fca7ee4e095a794727c0556d79
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,6 @@
[submodule "couchdb/couchdb-lucene"]
path = couchdb/couchdb-lucene
url = https://github.com/rnewson/couchdb-lucene.git
-[submodule "couchdb/settee"]
- path = couchdb/settee
- url = https://github.com/inadarei/settee.git
[submodule "lib/php-diff"]
path = lib/php-diff
url = https://github.com/chrisboulton/php-diff.git
--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -4,74 +4,62 @@
//function createFOIDocumentsDesignDoc() {
$foidb = $server->get_db('disclosr-foidocuments');
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
- $obj->views->byDate->reduce = "_count";
- $obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
- $obj->views->byAgencyID->reduce = "_count";
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
+$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
+$obj->views->byDate->reduce = "_count";
+$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";
+$obj->views->byDateMonthYear->reduce = "_count";
+$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
+$obj->views->byAgencyID->reduce = "_count";
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- $foidb->save($obj, true);
+// allow safe updates (even if slightly slower due to extra: rev-detection check).
+$foidb->save($obj, true);
-function createDocumentsDesignDoc() {
- /*
- global $db;
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
- "views": {
- "web_server": {
- "map": "function(doc) {\n emit(doc.web_server, 1);\n}",
- "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
- },
- "byAgency": {
- "map": "function(doc) {\n emit(doc.agencyID, 1);\n}",
- "reduce": "function (key, values, rereduce) {\n return sum(values);\n}"
- },
- "byURL": {
- "map": "function(doc) {\n emit(doc.url, doc);\n}"
- },
- "agency": {
- "map": "function(doc) {\n emit(doc.agencyID, doc);\n}"
- },
- "byWebServer": {
- "map": "function(doc) {\n emit(doc.web_server, doc);\n}"
- },
- "getValidationRequired": {
- "map": "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}"
- }
- } */
-}
+//function createDocumentsDesignDoc() {
+$docdb = $server->get_db('disclosr-documents');
+
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->web_server->map = "function(doc) {\n emit(doc.web_server, 1);\n}";
+$obj->views->web_server->reduce = "function (key, values, rereduce) {\n return sum(values);\n}";
+$obj->views->byAgency->map = "function(doc) {\n emit(doc.agencyID, 1);\n}";
+$obj->views->byAgency->reduce = "function (key, values, rereduce) {\n return sum(values);\n}";
+$obj->views->byURL->map = "function(doc) {\n emit(doc.url, doc);\n}";
+$obj->views->agency->map = "function(doc) {\n emit(doc.agencyID, doc);\n}";
+$obj->views->byWebServer->map = "function(doc) {\n emit(doc.web_server, doc);\n}";
+$obj->views->getValidationRequired = "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}";
+
+
+
//function createAgencyDesignDoc() {
$db = $server->get_db('disclosr-agencies');
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode("app");
- $obj->language = "javascript";
- $obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
- $obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
- $obj->views->byCanonicalName->map = "function(doc) {
+$obj = new stdClass();
+$obj->_id = "_design/" . urlencode("app");
+$obj->language = "javascript";
+$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
+$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
+$obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg || doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc);
}
};";
- $obj->views->byDeptStateName->map = "function(doc) {
+$obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id);
}
};";
- $obj->views->parentOrgs->map = "function(doc) {
+$obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) {
emit(doc._id, doc.parentOrg);
}
};";
- $obj->views->byName->map = 'function(doc) {
+$obj->views->byName->map = 'function(doc) {
if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") {
emit(doc.name, doc._id);
if (typeof(doc.shortName) != "undefined" && doc.shortName != doc.name) {
@@ -95,14 +83,14 @@
}
};';
- $obj->views->foiEmails->map = "function(doc) {
+$obj->views->foiEmails->map = "function(doc) {
emit(doc._id, doc.foiEmail);
};";
- $obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
- $obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
- $obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
- $obj->views->getScrapeRequired->map = "function(doc) {
+$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
+$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
+$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
+$obj->views->getScrapeRequired->map = "function(doc) {
var lastScrape = Date.parse(doc.metadata.lastScraped);
@@ -113,14 +101,14 @@
}
};";
- $obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
- $obj->views->getConflicts->map = "function(doc) {
+$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
+$obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts));
}
}";
- // http://stackoverflow.com/questions/646628/javascript-startswith
- $obj->views->score->map = 'if(!String.prototype.startsWith){
+// http://stackoverflow.com/questions/646628/javascript-startswith
+$obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {
return !this.indexOf(str);
}
@@ -144,7 +132,7 @@
emit(count+doc._id, {id:doc._id, name: doc.name, score:count, orgType: doc.orgType, portfolio:portfolio});
}
}';
- $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
+$obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {
return !this.indexOf(str);
}
@@ -164,22 +152,20 @@
emit("total", 1);
}
}';
- $obj->views->scoreHas->reduce = 'function (key, values, rereduce) {
+$obj->views->scoreHas->reduce = 'function (key, values, rereduce) {
return sum(values);
}';
- $obj->views->fieldNames->map = '
+$obj->views->fieldNames->map = '
function(doc) {
for(var propName in doc) {
emit(propName, doc._id);
}
}';
- $obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
+$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
return values.length;
}';
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- $db->save($obj, true);
-
-
+// allow safe updates (even if slightly slower due to extra: rev-detection check).
+$db->save($obj, true);
?>
--- a/couchdb/SetteeDatabase.class.php
+++ /dev/null
@@ -1,306 +1,1 @@
-<?php
-/**
-* Databaase class.
-*/
-class SetteeDatabase {
-
- /**
- * Base URL of the CouchDB REST API
- */
- private $conn_url;
-
- /**
- * HTTP REST Client instance
- */
- protected $rest_client;
-
- /**
- * Name of the database
- */
- private $dbname;
-
- /**
- * Default constructor
- */
- function __construct($conn_url, $dbname) {
- $this->conn_url = $conn_url;
- $this->dbname = $dbname;
- $this->rest_client = SetteeRestClient::get_instance($this->conn_url);
- }
-
-
- /**
- * Get UUID from CouchDB
- *
- * @return
- * CouchDB-generated UUID string
- *
- */
- function gen_uuid() {
- $ret = $this->rest_client->http_get('_uuids');
- return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking
- }
-
- /**
- * Create or update a document database
- *
- * @param $document
- * PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically.
- *
- * <p>If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation).
- * If "_id" is missing, CouchDB will be used to generate a UUID.
- *
- * <p>If $document has a "_rev" property (revision), document will be updated, rather than creating a new document.
- * You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be
- * one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but
- * not provide "_id" since that is an invalid input.
- *
- * @param $allowRevAutoDetection
- * Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision
- * for a document and use it. This option is "false" by default because it involves an extra http HEAD request and
- * therefore can make save() operation slightly slower if such auto-detection is not required.
- *
- * @return
- * document object with the database id (uuid) and revision attached;
- *
- * @throws SetteeCreateDatabaseException
- */
- function save($document, $allowRevAutoDetection = false) {
- if (is_string($document)) {
- $document = json_decode($document);
- }
-
- // Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter)
- if(is_array($document)) {
- $document = (object) $document;
- }
-
- if (empty($document->_id) && empty($document->_rev)) {
- $id = $this->gen_uuid();
- }
- elseif (empty($document->_id) && !empty($document->_rev)) {
- throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id");
- }
- else {
- $id = $document->_id;
-
- if ($allowRevAutoDetection) {
- try {
- $rev = $this->get_rev($id);
- } catch (SetteeRestClientException $e) {
- // auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error
- }
- if (!empty($rev)) {
- $document->_rev = $rev;
- }
- }
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $document_json = json_encode($document, JSON_NUMERIC_CHECK);
-
- $ret = $this->rest_client->http_put($full_uri, $document_json);
-
- $document->_id = $ret['decoded']->id;
- $document->_rev = $ret['decoded']->rev;
-
- return $document;
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $content
- * Content of the attachment in a string-buffer format. This function will automatically base64-encode content for
- * you, so you don't have to do it.
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment($doc, $name, $content, $mime_type = null) {
- if (empty($doc->_attachments) || !is_object($doc->_attachments)) {
- $doc->_attachments = new stdClass();
- }
-
- if (empty($mime_type)) {
- $mime_type = $this->rest_client->content_mime_type($content);
- }
-
- $doc->_attachments->$name = new stdClass();
- $doc->_attachments->$name->content_type = $mime_type;
- $doc->_attachments->$name->data = base64_encode($content);
- }
-
- /**
- * @param $doc
- * @param $name
- * @param $file
- * Full path to a file (e.g. as returned by PHP's realpath function).
- * @param $mime_type
- * Optional. Will be auto-detected if not provided
- * @return void
- */
- public function add_attachment_file($doc, $name, $file, $mime_type = null) {
- $content = file_get_contents($file);
- $this->add_attachment($doc, $name, $content, $mime_type);
- }
-
- /**
- *
- * Retrieve a document from CouchDB
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri);
- $ret = $this->rest_client->http_get($full_uri);
- return $ret['decoded'];
- }
-
- /**
- *
- * Get the latest revision of a document with document id: $id in CouchDB.
- *
- * @throws SetteeWrongInputException
- *
- * @param $id
- * Unique ID (usually: UUID) of the document to be retrieved.
- * @return
- * database document in PHP object format.
- */
- function get_rev($id) {
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't query a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
- $headers = $this->rest_client->http_head($full_uri);
- if (empty($headers['Etag'])) {
- throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag");
- }
- $etag = str_replace('"', '', $headers['Etag']);
- return $etag;
- }
-
- /**
- * Delete a document
- *
- * @param $document
- * a PHP object or JSON representation of the document that has _id and _rev fields.
- *
- * @return void
- */
- function delete($document) {
- if (!is_object($document)) {
- $document = json_decode($document);
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev;
- $this->rest_client->http_delete($full_uri);
- }
-
-
- /*----------------- View-related functions --------------*/
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $map_src
- * Source code of the map function in Javascript
- * @param $reduce_src
- * Source code of the reduce function in Javascript (optional)
- * @return void
- */
- function save_view($design_doc, $view_name, $map_src, $reduce_src = null) {
- $obj = new stdClass();
- $obj->_id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $obj->views->$view_name->map = $map_src;
- if (!empty($reduce_src)) {
- $obj->views->$view_name->reduce = $reduce_src;
- }
-
- // allow safe updates (even if slightly slower due to extra: rev-detection check).
- return $this->save($obj, true);
- }
-
- /**
- * Create a new view or update an existing one.
- *
- * @param $design_doc
- * @param $view_name
- * @param $key
- * key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes
- * that first element is startkey, second: endkey.
- * @param $descending
- * return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change
- * order you also need to swap startkey and endkey values!
- *
- * @return void
- */
- function get_view($design_doc, $view_name, $key = null, $descending = false) {
- $id = "_design/" . urlencode($design_doc);
- $view_name = urlencode($view_name);
- $id .= "/_view/$view_name";
-
- $data = array();
- if (!empty($key)) {
- if (is_string($key)) {
- $data = "key=" . '"' . $key . '"';
- }
- elseif (is_array($key)) {
- list($startkey, $endkey) = $key;
- $data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"';
- }
-
- if ($descending) {
- $data .= "&descending=true";
- }
- }
-
-
-
- if (empty($id)) {
- throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid.");
- }
-
- $full_uri = $this->dbname . "/" . $this->safe_urlencode($id);
-$full_uri = str_replace("%253Fgroup%253Dtrue","?group=true",$full_uri);
- $ret = $this->rest_client->http_get($full_uri, $data);
- return $ret['decoded'];
-
- }
-
- /**
- * @param $id
- * @return
- * return a properly url-encoded id.
- */
- private function safe_urlencode($id) {
- //-- System views like _design can have "/" in their URLs.
- $id = rawurlencode($id);
- if (substr($id, 0, 1) == '_') {
- $id = str_replace('%2F', '/', $id);
- }
- return $id;
- }
-
- /** Getter for a database name */
- function get_name() {
- return $this->dbname;
- }
-
-}
--- a/couchdb/settee
+++ /dev/null
--- a/documents/disclogsList.php
+++ b/documents/disclogsList.php
@@ -12,6 +12,7 @@
$disclogs = 0;
$red = 0;
$green = 0;
+$yellow = 0;
$orange = 0;
try {
$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;
@@ -46,8 +47,14 @@
echo "<font color='green'><abbr title='Yes'>✔</abbr></font>";
$green++;
} else if (file_exists("./scrapers/" . $row->id . '.txt')) {
+ if (trim(file_get_contents("./scrapers/" . $row->id . '.txt')) == "no disclog") {
+ echo "<font color='yellow'><abbr title='No log table exists at URL to scrape'><b>◎</b></abbr></font>";
+ $yellow++;
+ } else {
+ echo file_get_contents("./scrapers/" . $row->id . '.txt');
echo "<font color='orange'><abbr title='Work in progress'><b>▬</b></abbr></font>";
$orange++;
+ }
} else {
echo "<font color='red'><abbr title='No'>✘</abbr></font>";
$red++;
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -9,6 +9,7 @@
import dateutil
from dateutil.parser import *
from datetime import *
+import codecs
class GenericDisclogScraper(object):
__metaclass__ = abc.ABCMeta
@@ -93,7 +94,7 @@
def getDate(self, content, entry, doc):
date = ''.join(content.stripped_strings).strip()
(a,b,c) = date.partition("(")
- date = a.replace("Octber","October")
+ date = self.remove_control_chars(a.replace("Octber","October"))
print date
edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d")
print edate
@@ -120,7 +121,7 @@
columns = row.find_all('td')
if len(columns) == self.getColumnCount():
(id, date, title, description, notes) = self.getColumns(columns)
- print ''.join(id.stripped_strings)
+ print self.remove_control_chars(''.join(id.stripped_strings))
if id.string == None:
hash = scrape.mkhash(self.remove_control_chars(url+(''.join(date.stripped_strings))))
else:
--- a/documents/index.php
+++ b/documents/index.php
@@ -1,11 +1,11 @@
<?php
+
include('template.inc.php');
include_header_documents("");
include_once('../include/common.inc.php');
+$startkey = (isset($_REQUEST['start_key']) ? $_REQUEST['start_key'] : '9999-99-99');
?>
<?php
-
-
$agenciesdb = $server->get_db('disclosr-agencies');
@@ -15,17 +15,17 @@
}
$foidocsdb = $server->get_db('disclosr-foidocuments');
try {
- $rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00'), true)->rows;
-
-
+ $rows = $foidocsdb->get_view("app", "byDate", Array($startkey, '0000-00-00'), true, 20)->rows;
if ($rows) {
- foreach ($rows as $row) {
-echo displayLogEntry($row,$idtoname);
+ foreach ($rows as $key => $row) {
+ echo displayLogEntry($row, $idtoname);
+ $endkey = $row->key;
}
}
} catch (SetteeRestClientException $e) {
setteErrorHandler($e);
}
+echo "<a href='?start_key=$endkey'>next page</a>";
include_footer_documents();
?>
--- a/documents/scrapers/627f116dfe42c9f27ad6747be0aa44e2.txt
+++ b/documents/scrapers/627f116dfe42c9f27ad6747be0aa44e2.txt
@@ -1,2 +1,1 @@
-see parent dhs
-
+no disclog
--- a/documents/scrapers/7c6adc1d41cf029bf1a0959e5156477a.txt
+++ b/documents/scrapers/7c6adc1d41cf029bf1a0959e5156477a.txt
@@ -1,1 +1,1 @@
-
+acma style
--- /dev/null
+++ b/documents/scrapers/8e874a2fde8aa0ccdc6d14573d766540.py
@@ -1,1 +1,86 @@
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import genericScrapers
+import scrape
+from bs4 import BeautifulSoup
+import codecs
+#http://www.doughellmann.com/PyMOTW/abc/
+class NewScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
+ def getDescription(self,content, entry,doc):
+ link = None
+ links = []
+ description = ""
+ for atag in entry.find_all('a'):
+ if atag.has_key('href'):
+ link = scrape.fullurl(self.getURL(),atag['href'])
+ (url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
+ if htcontent != None:
+ if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+ # http://www.crummy.com/software/BeautifulSoup/documentation.html
+ soup = BeautifulSoup(htcontent)
+ for text in soup.find(id="divFullWidthColumn").stripped_strings:
+ description = description + text.encode('ascii', 'ignore')
+
+ for atag in soup.find(id="divFullWidthColumn").find_all("a"):
+ if atag.has_key('href'):
+ links.append(scrape.fullurl(link,atag['href']))
+ if links != []:
+ doc.update({'links': links})
+ if description != "":
+ doc.update({ 'description': description})
+
+ def getColumnCount(self):
+ return 2
+ def getTable(self,soup):
+ return soup.find(id = "TwoColumnSorting")
+ def getColumns(self,columns):
+ ( title, date) = columns
+ return (title, date, title, title, None)
+class OldScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
+ def getDescription(self,content, entry,doc):
+ link = None
+ links = []
+ description = ""
+ for atag in entry.find_all('a'):
+ if atag.has_key('href'):
+ link = scrape.fullurl(self.getURL(),atag['href'])
+ (url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
+ if htcontent != None:
+ if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+ # http://www.crummy.com/software/BeautifulSoup/documentation.html
+ soup = BeautifulSoup(htcontent)
+ for text in soup.find(id="content-item").stripped_strings:
+ description = description + text + " \n"
+ for atag in soup.find(id="content-item").find_all("a"):
+ if atag.has_key('href'):
+ links.append(scrape.fullurl(link,atag['href']))
+ if links != []:
+ doc.update({'links': links})
+ if description != "":
+ doc.update({ 'description': description})
+
+ if links != []:
+ doc.update({'links': links})
+ if description != "":
+ doc.update({ 'description': description})
+
+ def getColumnCount(self):
+ return 2
+ def getTable(self,soup):
+ return soup.find(class_ = "doc-list")
+ def getColumns(self,columns):
+ (date, title) = columns
+ return (title, date, title, title, None)
+
+if __name__ == '__main__':
+ print 'Subclass:', issubclass(NewScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
+ print 'Instance:', isinstance(NewScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
+ #NewScraperImplementation().doScrape()
+ print 'Subclass:', issubclass(OldScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
+ print 'Instance:', isinstance(OldScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
+ osi = OldScraperImplementation()
+ osi.disclogURL = "http://archive.treasury.gov.au/content/foi_publications.asp?year=-1&abstract=0&classification=&=&titl=Disclosure+Log+-+Documents+Released+Under+FOI"
+ osi.doScrape()
+# old site too
+
--- a/documents/scrapers/8e874a2fde8aa0ccdc6d14573d766540.txt
+++ /dev/null
@@ -1,49 +1,1 @@
-import sys,os
-sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
-import genericScrapers
-import scrape
-from bs4 import BeautifulSoup
-#http://www.doughellmann.com/PyMOTW/abc/
-class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
- def getDescription(self,content, entry,doc):
- link = None
- links = []
- description = ""
- for atag in entry.find_all('a'):
- if atag.has_key('href'):
- link = scrape.fullurl(self.getURL(),atag['href'])
- (url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False)
- if htcontent != None:
- if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
- # http://www.crummy.com/software/BeautifulSoup/documentation.html
- soup = BeautifulSoup(htcontent)
- for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'):
- if row != None:
- rowtitle = row.find('th').string
- description = description + "\n" + rowtitle + ": "
- for text in row.find('td').stripped_strings:
- description = description + text
- for atag in row.find_all("a"):
- if atag.has_key('href'):
- links.append(scrape.fullurl(link,atag['href']))
-
- if links != []:
- doc.update({'links': links})
- if description != "":
- doc.update({ 'description': description})
-
- def getColumnCount(self):
- return 2
- def getTable(self,soup):
- return soup.find(class_ = "ms-rteTable-GreyAlternating")
- def getColumns(self,columns):
- (date, title) = columns
- return (title, date, title, title, None)
-
-if __name__ == '__main__':
- print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
- print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
- ScraperImplementation().doScrape()
-# old site too http://archive.treasury.gov.au/content/foi_publications.asp
-
--- a/documents/scrapers/bb96fe4065afb7e0872136dd657f9369.txt
+++ b/documents/scrapers/bb96fe4065afb7e0872136dd657f9369.txt
@@ -1,2 +1,1 @@
-# does not have any disclog entries or table
-
+no disclog
--- a/documents/scrapers/bf6e587f166040b63681cd2ff76fbfdf.txt
+++ b/documents/scrapers/bf6e587f166040b63681cd2ff76fbfdf.txt
@@ -1,1 +1,1 @@
-no disclog yet
+no disclog
--- a/documents/scrapers/cb7f40e3495b682de6eee61bf09c1cfc.txt
+++ b/documents/scrapers/cb7f40e3495b682de6eee61bf09c1cfc.txt
@@ -1,2 +1,1 @@
-no log
-
+no disclog
--- a/documents/scrapers/d72744fb1e5d6e87af9a5ea16cc27fa5.txt
+++ b/documents/scrapers/d72744fb1e5d6e87af9a5ea16cc27fa5.txt
@@ -1,1 +1,1 @@
-
+acma style
--- a/documents/scrapers/e770921522a49dc77de208cc724ce134.txt
+++ b/documents/scrapers/e770921522a49dc77de208cc724ce134.txt
@@ -1,2 +1,1 @@
-c'est ne pas une table
-
+no disclog
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -145,7 +145,7 @@
$result .= "</ul>";
}
- $result .= "<small><A href='".$row->value->url."'>View original source...</a> ID: ".$row->value->docID."</small>";
+ $result .= "<small><A href='".$row->value->url."'>View original source...</a> ID: ".strip_tags($row->value->docID)."</small>";
$result .= "</div>";
return $result;
}