upgrade scrapers to beautifulsoup4
Former-commit-id: 350a28ba24aef1eb018e2db0275aa62fe3369728
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,16 +10,22 @@
[submodule "lib/Requests"]
path = lib/Requests
url = https://github.com/rmccue/Requests.git
-[submodule "javascripts/flotr2"]
- path = javascripts/flotr2
+[submodule "js/flotr2"]
+ path = js/flotr2
url = https://github.com/HumbleSoftware/Flotr2.git
[submodule "lib/phpquery"]
path = lib/phpquery
url = https://github.com/TobiaszCudnik/phpquery.git
-[submodule "javascripts/sigma"]
- path = javascripts/sigma
+[submodule "js/sigma"]
+ path = js/sigma
url = https://github.com/jacomyal/sigma.js.git
-[submodule "javascripts/bubbletree"]
- path = javascripts/bubbletree
+[submodule "js/bubbletree"]
+ path = js/bubbletree
url = https://github.com/okfn/bubbletree.git
+[submodule "lib/querypath"]
+ path = lib/querypath
+ url = https://github.com/technosophos/querypath.git
+[submodule "lib/amon-php"]
+ path = lib/amon-php
+ url = https://github.com/martinrusev/amon-php.git
--- a/about.php
+++ b/about.php
@@ -8,10 +8,12 @@
</div>
<h2> What is this? </h2>
Disclo.gs is a project to monitor Australian Federal Government agencies
-compliance with their <a href="http://www.oaic.gov.au/publications/other_operational/foi_policy_frequently_asked_questions.html#_Toc291837571">"proactive disclosure requirements"</a>.
+compliance with their <a href="http://www.oaic.gov.au/publications/other_operational/foi_policy_frequently_asked_questions.html#_Toc291837571">"proactive disclosure requirements" to make a transparency league table as suggested by gov2 taskforce http://gov2.net.au/blog/2009/09/19/a-league-ladder-of-psi-openness/</a>.
<h2> Attributions </h2>
-National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm.
-
+National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm <br/>
+data.gov.au http://data.gov.au/dataset/directory-gov-au-full-data-export/ <br/>
+directory.gov.au <br/>
+australia.gov.au http://australia.gov.au/about/copyright <br/>
<h2> Open everything </h2>
All documents released CC-BY 3 AU
Open source git @
--- /dev/null
+++ b/admin/agls.php
@@ -1,1 +1,35 @@
+<?php
+include_once('include/common.inc.php');
+include_header('Webserver and Accessiblity');
+
+echo "<table>
+ <tr><th>name</th><th>function</th></tr>";
+$db = $server->get_db('disclosr-agencies');
+try {
+ $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+
+
+ if ($agencies) {
+ foreach ($agencies as $row) {
+
+ echo "<tr><td>" . $row->value->name . "</td>";
+ if (isset($row->value->metaTags)) {
+ if (is_array($row->value->metaTags)) {
+ $tags =$row->value->metaTags;
+ } else {
+ $tags = object_to_array($row->value->metaTags);
+ }
+ if (isset($tags['AGLS.Function'])) {
+ echo "<td>" . $tags['AGLS.Function'] . "</td>";
+ }
+ }
+ echo "</tr>";
+ }
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+include_footer();
+?>
+
--- a/admin/conflicts.php
+++ b/admin/conflicts.php
@@ -1,7 +1,7 @@
<?php
include_once('../include/common.inc.php');
-include_header();
+include_header('Fix Conflicts');
require_once '../lib/php-diff/lib/Diff.php';
require_once '../lib/php-diff/lib/Diff/Renderer/Html/SideBySide.php';
--- /dev/null
+++ b/admin/genericAgencyFixer.php
@@ -1,1 +1,44 @@
+<?php
+include_once("../include/common.inc.php");
+require($basePath . 'lib/phpquery/phpQuery/phpQuery.php');
+
+setlocale(LC_CTYPE, 'C');
+
+
+$db = $server->get_db('disclosr-agencies');
+
+try {
+ $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+ //print_r($rows);
+ foreach ($agencies as $agency) {
+ //echo $agency->value->name . " ".$agency->value->website."<br />\n";
+ // print_r($agency);
+ //hasRestricitiveLicence" hasRestrictiveLicense -> has Restrictive Licence
+ // "hasYoutube" -> Tube
+ // "comment" -> "comments"
+ if (!isset($agency->value->metaTags) && isset($agency->value->website)) {
+ echo $agency->value->name . " ".$agency->value->website."<br />\n";
+ $agency->value->metaTags = Array();
+ $request = Requests::get($agency->value->website);
+ $html = phpQuery::newDocumentHTML($request->body);
+ phpQuery::selectDocument($html);
+ foreach (pq('meta')->elements as $meta) {
+ $tagName = $meta->getAttribute('name');;
+ $content = $meta->getAttribute('content');
+ if ($tagName != "") {
+echo "$tagName == $content <br>\n";
+ $agency->value->metaTags[$tagName] = $content;
+ }
+ }
+ //print_r($agency->value->metaTags);
+ $db->save($agency->value);
+ echo "<hr>";
+ flush();
+ }
+ }
+} catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+}
+?>
+
--- /dev/null
+++ b/admin/importRTKbodies.php
@@ -1,1 +1,56 @@
+<?php
+require_once '../include/common.inc.php';
+
+$db = $server->get_db('disclosr-agencies');
+$rows = $db->get_view("app", "byName")->rows;
+$nametoid = Array();
+$accounts = Array();
+foreach ($rows as $row) {
+ $nametoid[trim($row->key)] = $row->value;
+}
+
+function extractCSVAccounts($url, $nameField, $accountField, $filter) {
+ global $accounts, $nametoid;
+ $request = Requests::get($url);
+ echo $url;
+ $Data = str_getcsv($request->body, "\n"); //parse the rows
+ $headers = Array();
+ foreach ($Data as $num => $line) {
+ $Row = str_getcsv($line, ",");
+ if ($num == 0) {
+ $headers = $Row;
+ print_r($headers);
+ } else {
+ if (isset($Row[array_search($nameField, $headers)])) {
+ $agencyName = $Row[array_search($nameField, $headers)];
+ if (!in_array(trim($agencyName), array_keys($nametoid))) {
+ echo "$agencyName missing" . PHP_EOL;
+ } else {
+ echo $Row[array_search($nameField, $headers)] . PHP_EOL;
+ $accounts[$nametoid[trim($agencyName)]]["rtkURLs"][$agencyName] = 'http://www.righttoknow.org.au/body/'.$Row[array_search($accountField, $headers)];
+ }
+ } else {
+ echo "error finding any agency" . $line . PHP_EOL;
+ }
+ }
+ }
+}
+
+extractCSVAccounts("http://www.righttoknow.org.au/body/all-authorities.csv","Agency","URL name");
+print_r($accounts);
+/* foreach ($accounts as $id => $accountTypes) {
+ echo $id . "<br>" . PHP_EOL;
+ $doc = object_to_array($db->get($id));
+ // print_r($doc);
+
+ foreach ($accountTypes as $accountType => $accounts) {
+ if (!isset($doc["has" . $accountType]) || !is_array($doc["has" . $accountType])) {
+ $doc["has" . $accountType] = Array();
+ }
+ $doc["has" . $accountType] = array_unique(array_merge($doc["has" . $accountType], $accounts));
+ }
+ $db->save($doc);
+}*/
+?>
+
--- a/admin/naa-agift-tree.php
+++ b/admin/naa-agift-tree.php
@@ -31,9 +31,9 @@
<div id="tree-container"></div>
<div id="viz"></div>
-<script type="text/javascript" src="../javascripts/jquery-1.6.2.min.js"></script>
-<script type="text/javascript" src="../javascripts/d3.min.js"></script>
-<script type="text/javascript" src="../javascripts/d3.layout.min.js"></script>
+<script type="text/javascript" src="../js/jquery-1.6.2.min.js"></script>
+<script type="text/javascript" src="../js/d3.min.js"></script>
+<script type="text/javascript" src="../js/d3.layout.min.js"></script>
<script>
/**
--- a/alaveteli/exportAgencies.csv.php
+++ b/alaveteli/exportAgencies.csv.php
@@ -1,20 +1,6 @@
<?php
include_once("../include/common.inc.php");
-
-function shortName($name) {
- $name = trim($name);
- if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
- $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
- return str_replace($badWords,"",$name);
- }
-
- else {
- $out = Array();
- preg_match_all('/[A-Z]/', $name, $out);
- return implode("", $out[0]);
- }
-}
setlocale(LC_CTYPE, 'C');
@@ -63,38 +49,36 @@
$row = Array();
$row["#id"] = $agency->id;
$row["name"] = trim($agency->value->name);
- if (isset($agency->value->foiEmail)) {
- $row["request_email"] = $agency->value->foiEmail;
- } else {
- if ($agency->value->orgType == "FMA-DepartmentOfState") {
- $row["request_email"] = "foi@" . GetDomain($agency->value->website);
- } else {
- $row["request_email"] = $foiEmail[$agency->value->parentOrg];
+ $row["request_email"] = (isset($agency->value->foiEmail) ? $agency->value->foiEmail : "");
+ $row["short_name"] = (isset($agency->value->shortName) ? $agency->value->shortName : "");
+ $row["notes"] = (isset($agency->value->description) ? $agency->value->description : "");
+
+ $otherBodies = Array();
+ if (isset($agency->value->foiBodies)) {
+ $otherBodies = array_merge($otherBodies, $agency->value->foiBodies);
+ }
+ if (isset($agency->value->positions)) {
+ $positions = Array();
+ foreach ($agency->value->positions as $position) {
+ $positions[] = "Office of the ".$position;
}
+ $otherBodies = array_merge($otherBodies, $positions);
}
- if (isset($agency->value->shortName)) {
- $row["short_name"] = $agency->value->shortName;
- } else {
- $row["short_name"] = shortName($agency->value->name);
+ sort($otherBodies);
+ if (count($otherBodies) > 0) {
+ $row["notes"] .= "<br/> This department also responds to requests for information held by " . implode(", ", $otherBodies);
}
- $row["notes"] = "";
+
$row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
$row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
if ($agency->value->orgType == "FMA-DepartmentOfState") {
- $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
+ $row["tag_string"] = $tag[$agency->value->_id];
} else {
- $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
+ $row["tag_string"] = $tag[$agency->value->parentOrg];
}
-
+ $row["tag_string"] .= " " . $agency->value->orgType;
+ $row["tag_string"] .= " federal";
fputcsv($fp, array_values($row));
-
- if (isset($agency->value->foiBodies)) {
- foreach ($agency->value->foiBodies as $foiBody) {
- $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
- $row["short_name"] = shortName($foiBody);
- fputcsv($fp, array_values($row));
- }
- }
}
}
} catch (SetteeRestClientException $e) {
--- a/bubbletree.php
+++ b/bubbletree.php
@@ -5,13 +5,13 @@
<meta charset="UTF-8"/>
<title>Minimal BubbleTree Demo</title>
<script type="text/javascript" src="http://code.jquery.com/jquery-1.7.2.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/jquery.history.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/raphael.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/vis4.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/lib/Tween.js"></script>
- <script type="text/javascript" src="javascripts/bubbletree/build/bubbletree.js"></script>
- <link rel="stylesheet" type="text/css" href="javascripts/bubbletree/build/bubbletree.css" />
- <script type="text/javascript" src="javascripts/bubbletree/styles/cofog.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/jquery.history.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/raphael.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/vis4.js"></script>
+ <script type="text/javascript" src="js/bubbletree/lib/Tween.js"></script>
+ <script type="text/javascript" src="js/bubbletree/build/bubbletree.js"></script>
+ <link rel="stylesheet" type="text/css" href="js/bubbletree/build/bubbletree.css" />
+ <script type="text/javascript" src="js/bubbletree/styles/cofog.js"></script>
<script type="text/javascript">
--- /dev/null
+++ b/documents/google676a414ad086cefb.html
@@ -1,1 +1,2 @@
+google-site-verification: google676a414ad086cefb.html
--- /dev/null
+++ b/documents/index.php
@@ -1,1 +1,15 @@
+australian disclosure logs
+are you looking for more information about:
+contracts
+gov orgs
+lobbyists
+
+1/1/11 title (Dept dfggdfgdf)
+description:
+source link:
+documents:
+ #1 title link
+
+
+rss feed here
--- /dev/null
+++ b/documents/rss.xml.php
@@ -1,1 +1,30 @@
+<?php
+// Agency X updated Y, new files, diff of plain text/link text,
+// feed for just one agency or all
+// This is a minimum example of using the Universal Feed Generator Class
+include("lib/FeedWriter.php");
+//Creating an instance of FeedWriter class.
+$TestFeed = new FeedWriter(RSS2);
+//Setting the channel elements
+//Use wrapper functions for common channelelements
+$TestFeed->setTitle('Last Modified - All');
+$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php');
+$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
+//Retriving informations from database
+$rows = $db->get_view("app", "byLastModified")->rows;
+//print_r($rows);
+foreach ($rows as $row) {
+ //Create an empty FeedItem
+ $newItem = $TestFeed->createNewItem();
+ //Add elements to the feed item
+ $newItem->setTitle($row['name']);
+ $newItem->setLink($row['id']);
+ $newItem->setDate(date("c", $row['metadata']['lastModified']));
+ $newItem->setDescription($row['name']);
+ //Now add the feed item
+ $TestFeed->addItem($newItem);
+}
+//OK. Everything is done. Now genarate the feed.
+$TestFeed->genarateFeed();
+?>
--- /dev/null
+++ b/documents/runScrapers.php
--- /dev/null
+++ b/documents/scrape.py
@@ -1,1 +1,204 @@
-
+#http://packages.python.org/CouchDB/client.html
+import couchdb
+import urllib2
+from BeautifulSoup import BeautifulSoup
+import re
+import hashlib
+from urlparse import urljoin
+import time
+import os
+import mimetypes
+import re
+import urllib
+import urlparse
+
+def canonurl(url):
+ r"""Return the canonical, ASCII-encoded form of a UTF-8 encoded URL, or ''
+ if the URL looks invalid.
+ >>> canonurl('\xe2\x9e\xa1.ws') # tinyarro.ws
+ 'http://xn--hgi.ws/'
+ """
+ # strip spaces at the ends and ensure it's prefixed with 'scheme://'
+ url = url.strip()
+ if not url:
+ return ''
+ if not urlparse.urlsplit(url).scheme:
+ url = 'http://' + url
+
+ # turn it into Unicode
+ #try:
+ # url = unicode(url, 'utf-8')
+ #except UnicodeDecodeError:
+ # return '' # bad UTF-8 chars in URL
+
+ # parse the URL into its components
+ parsed = urlparse.urlsplit(url)
+ scheme, netloc, path, query, fragment = parsed
+
+ # ensure scheme is a letter followed by letters, digits, and '+-.' chars
+ if not re.match(r'[a-z][-+.a-z0-9]*$', scheme, flags=re.I):
+ return ''
+ scheme = str(scheme)
+
+ # ensure domain and port are valid, eg: sub.domain.<1-to-6-TLD-chars>[:port]
+ match = re.match(r'(.+\.[a-z0-9]{1,6})(:\d{1,5})?$', netloc, flags=re.I)
+ if not match:
+ return ''
+ domain, port = match.groups()
+ netloc = domain + (port if port else '')
+ netloc = netloc.encode('idna')
+
+ # ensure path is valid and convert Unicode chars to %-encoded
+ if not path:
+ path = '/' # eg: 'http://google.com' -> 'http://google.com/'
+ path = urllib.quote(urllib.unquote(path.encode('utf-8')), safe='/;')
+
+ # ensure query is valid
+ query = urllib.quote(urllib.unquote(query.encode('utf-8')), safe='=&?/')
+
+ # ensure fragment is valid
+ fragment = urllib.quote(urllib.unquote(fragment.encode('utf-8')))
+
+ # piece it all back together, truncating it to a maximum of 4KB
+ url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
+ return url[:4096]
+
+#http://diveintopython.org/http_web_services/etags.html
+class NotModifiedHandler(urllib2.BaseHandler):
+ def http_error_304(self, req, fp, code, message, headers):
+ addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
+ addinfourl.code = code
+ return addinfourl
+
+def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True):
+ url = canonurl(url)
+ hash = hashlib.md5(url).hexdigest().encode("utf-8")
+ req = urllib2.Request(url)
+ print "Fetching %s" % url
+ if url.startswith("mailto") or url.startswith("javascript") or url.startswith("#") or url == None or url == "":
+ print "Not a valid HTTP url"
+ return (None,None,None)
+ doc = docsdb.get(hash)
+ if doc == None:
+ doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName}
+ else:
+ if (('page_scraped' in doc) and (time.time() - doc['page_scraped']) < 999999):
+ print "Uh oh, trying to scrape URL again too soon!"
+ last_attachment_fname = doc["_attachments"].keys()[-1]
+ last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
+ return (doc['url'],doc['mime_type'],last_attachment.read())
+ if scrape_again == False:
+ print "Not scraping this URL again as requested"
+ return (None,None,None)
+
+ time.sleep(3) # wait 3 seconds to give webserver time to recover
+
+ req.add_header("User-Agent", "Mozilla/4.0 (compatible; Prometheus webspider; owner maxious@lambdacomplex.org)")
+ #if there is a previous version stored in couchdb, load caching helper tags
+ if doc.has_key('etag'):
+ req.add_header("If-None-Match", doc['etag'])
+ if doc.has_key('last_modified'):
+ req.add_header("If-Modified-Since", doc['last_modified'])
+
+ opener = urllib2.build_opener(NotModifiedHandler())
+ try:
+ url_handle = opener.open(req)
+ doc['url'] = url_handle.geturl() # may have followed a redirect to a new url
+ headers = url_handle.info() # the addinfourls have the .info() too
+ doc['etag'] = headers.getheader("ETag")
+ doc['last_modified'] = headers.getheader("Last-Modified")
+ doc['date'] = headers.getheader("Date")
+ doc['page_scraped'] = time.time()
+ doc['web_server'] = headers.getheader("Server")
+ doc['via'] = headers.getheader("Via")
+ doc['powered_by'] = headers.getheader("X-Powered-By")
+ doc['file_size'] = headers.getheader("Content-Length")
+ content_type = headers.getheader("Content-Type")
+ if content_type != None:
+ doc['mime_type'] = content_type.split(";")[0]
+ else:
+ (type,encoding) = mimetypes.guess_type(url)
+ doc['mime_type'] = type
+ if hasattr(url_handle, 'code'):
+ if url_handle.code == 304:
+ print "the web page has not been modified"
+ return (None,None,None)
+ else:
+ content = url_handle.read()
+ docsdb.save(doc)
+ doc = docsdb.get(hash) # need to get a _rev
+ docsdb.put_attachment(doc, content, str(time.time())+"-"+os.path.basename(url), doc['mime_type'])
+ return (doc['url'], doc['mime_type'], content)
+ #store as attachment epoch-filename
+ except urllib2.URLError as e:
+ error = ""
+ if hasattr(e, 'reason'):
+ error = "error %s in downloading %s" % (str(e.reason), url)
+ elif hasattr(e, 'code'):
+ error = "error %s in downloading %s" % (e.code, url)
+ print error
+ doc['error'] = error
+ docsdb.save(doc)
+ return (None,None,None)
+
+
+
+def scrapeAndStore(docsdb, url, depth, fieldName, agencyID):
+ (url,mime_type,content) = fetchURL(docsdb, url, fieldName, agencyID)
+ badURLs = ["http://www.ausport.gov.au/supporting/funding/grants_and_scholarships/grant_funding_report"]
+ if content != None and depth > 0 and url != "http://www.ausport.gov.au/supporting/funding/grants_and_scholarships/grant_funding_report":
+ if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+ # http://www.crummy.com/software/BeautifulSoup/documentation.html
+ soup = BeautifulSoup(content)
+ navIDs = soup.findAll(id=re.compile('nav|Nav|menu|bar|left|right|sidebar|more-links|breadcrumb|footer|header'))
+ for nav in navIDs:
+ print "Removing element", nav['id']
+ nav.extract()
+ navClasses = soup.findAll(attrs={'class' : re.compile('nav|menu|bar|left|right|sidebar|more-links|breadcrumb|footer|header')})
+ for nav in navClasses:
+ print "Removing element", nav['class']
+ nav.extract()
+ links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-"))
+ linkurls = set([])
+ for link in links:
+ if link.has_key("href"):
+ if link['href'].startswith("http"):
+ # lets not do external links for now
+ # linkurls.add(link['href'])
+ None
+ if link['href'].startswith("mailto"):
+ # not http
+ None
+ if link['href'].startswith("javascript"):
+ # not http
+ None
+ else:
+ # remove anchors and spaces in urls
+ link['href'] = link['href'].replace(" ","%20")
+ link['href'] = re.sub('#.*$','',link['href'])
+ linkurls.add(urljoin(url,link['href']))
+ for linkurl in linkurls:
+ #print linkurl
+ scrapeAndStore(docsdb, linkurl, depth-1, fieldName, agencyID)
+
+couch = couchdb.Server('http://127.0.0.1:5984/')
+# select database
+agencydb = couch['disclosr-agencies']
+docsdb = couch['disclosr-documents']
+
+if __name__ == "__main__":
+ for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view?
+ agency = agencydb.get(row.id)
+ print agency['name']
+ for key in agency.keys():
+ if key == 'website':
+ scrapeAndStore(docsdb, agency[key],0,key,agency['_id'])
+ if key.endswith('URL'):
+ print key
+ depth = 1
+ if 'scrapeDepth' in agency.keys():
+ depth = agency['scrapeDepth']
+ scrapeAndStore(docsdb, agency[key],depth,key,agency['_id'])
+ agency['metadata']['lastScraped'] = time.time()
+ agencydb.save(agency)
+
--- /dev/null
+++ b/documents/scrapers/3cd40b1240e987cbcd3f0e67054ce259.py
@@ -1,1 +1,24 @@
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import scrape
+foidocsdb = scrape.couch['disclosr-foidocuments']
+#RSS feed not detailed
+from bs4 import BeautifulSoup
+#http://www.apvma.gov.au/about/foi/disclosure/index.php
+(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, "http://www.apvma.gov.au/about/foi/disclosure/index.php", "foidocuments", "3cd40b1240e987cbcd3f0e67054ce259")
+if content != None:
+ if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
+ # http://www.crummy.com/software/BeautifulSoup/documentation.html
+ soup = BeautifulSoup(content)
+ for row in soup.table.find_all('tr'):
+ columns = row.find_all('td')
+ if len(columns) == 5:
+ (id, date, description, title, notes) = columns
+ print id
+ elif len(row.find_all('th')) == 5:
+ print "header row"
+ else:
+ print "ERROR number of columns incorrect"
+ print row
+
--- /dev/null
+++ b/documents/scrapers/8c9421f852c441910bf1d93a57b31d64.py
@@ -1,1 +1,8 @@
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import scrape
+foidocsdb = scrape.couch['disclosr-foidocuments']
+#rss feed has only one entry
+http://www.daff.gov.au/about/foi/ips/disclosure-log
+
--- /dev/null
+++ b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
@@ -1,1 +1,11 @@
+import sys,os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
+import scrape
+foidocsdb = scrape.couch['disclosr-foidocuments']
+import feedparser
+feed = feedparser.parse( "http://foi.deewr.gov.au/disclosure-log/rss")
+print feed.entries[0]
+#foreach feed.entries
+
+
--- /dev/null
+++ b/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
@@ -1,1 +1,1 @@
-
+www.finance.gov.au/foi/disclosure-log/foi-rss.xml
--- /dev/null
+++ b/documents/scrapers/rtk.py
@@ -1,1 +1,1 @@
-
+http://www.righttoknow.org.au/feed/search/%20(latest_status:successful%20OR%20latest_status:partially_successful)
--- /dev/null
+++ b/documents/search.php
@@ -1,1 +1,25 @@
+<?php
+include_once('include/common.inc.php�