gitphp 0.2.9.1 :: disclosr.git/commitdiff

beginning deewr scraper

Former-commit-id: c17beeeab98ca8e389303cfcc75566c09aaf49cc

3 files changed: (show all)
documents/genericScrapers.py
documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
documents/template.inc.php

file:a/documents/genericScrapers.py -> file:b/documents/genericScrapers.py

import sys,os	import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import scrape	import scrape
from bs4 import BeautifulSoup	from bs4 import BeautifulSoup
import parsedatetime as pdt	import parsedatetime as pdt
from time import mktime	from time import mktime
from datetime import datetime	from datetime import datetime
import feedparser	import feedparser
import abc	import abc

class GenericRSSDisclogScraper(object):	class GenericRSSDisclogScraper(object):
__metaclass__ = abc.ABCMeta	__metaclass__ = abc.ABCMeta
@abc.abstractmethod	@abc.abstractmethod
def getAgencyID(self):	def getAgencyID(self):
""" disclosr agency id """	""" disclosr agency id """
return	return

@abc.abstractmethod	@abc.abstractmethod
def getURL(self):	def getURL(self):
""" disclog URL"""	""" disclog URL"""
return	return

	def getDescription(self, entry, doc):
	""" get description from rss entry"""
	doc['description'] = entry.summary
	return

def doScrape(self):	def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())	(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
feed = feedparser.parse(content)	feed = feedparser.parse(content)
for entry in feed.entries:	for entry in feed.entries:
#print entry	#print entry
print entry.id	print entry.id
hash = scrape.mkhash(entry.link)	hash = scrape.mkhash(entry.id)
	#print hash
doc = foidocsdb.get(hash)	doc = foidocsdb.get(hash)
	#print doc
if doc == None:	if doc == None:
print "saving"	print "saving"
edate = datetime.fromtimestamp(mktime( entry.published_parsed)).strftime("%Y-%m-%d")	edate = datetime.fromtimestamp(mktime( entry.published_parsed)).strftime("%Y-%m-%d")
doc = {'id': hash, 'agencyID': self.getAgencyID(), 'url': entry.link, 'docID': entry.id,	doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': entry.link, 'docID': entry.id,
"date": edate, "description": entry.summary,"title": entry.title}	"date": edate,"title": entry.title}
	self.getDescription(entry, doc)
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved"	print "already saved"

class GenericOAICDisclogScraper(object):	class GenericOAICDisclogScraper(object):
__metaclass__ = abc.ABCMeta	__metaclass__ = abc.ABCMeta
@abc.abstractmethod	@abc.abstractmethod
def getAgencyID(self):	def getAgencyID(self):
""" disclosr agency id """	""" disclosr agency id """
return	return

@abc.abstractmethod	@abc.abstractmethod
def getURL(self):	def getURL(self):
""" disclog URL"""	""" disclog URL"""
return	return

@abc.abstractmethod	@abc.abstractmethod
def getColumns(self,columns):	def getColumns(self,columns):
""" rearranges columns if required """	""" rearranges columns if required """
return	return

def doScrape(self):	def doScrape(self):
cal = pdt.Calendar()	cal = pdt.Calendar()
foidocsdb = scrape.couch['disclosr-foidocuments']	foidocsdb = scrape.couch['disclosr-foidocuments']
(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())	(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
if content != None:	if content != None:
if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
# http://www.crummy.com/software/BeautifulSoup/documentation.html	# http://www.crummy.com/software/BeautifulSoup/documentation.html
soup = BeautifulSoup(content)	soup = BeautifulSoup(content)
for row in soup.table.find_all('tr'):	for row in soup.table.find_all('tr'):
columns = row.find_all('td')	columns = row.find_all('td')
if len(columns) == 5:	if len(columns) == 5:
(id, date, description, title, notes) = self.getColumns(columns)	(id, date, description, title, notes) = self.getColumns(columns)
print id.string	print id.string
hash = scrape.mkhash(url+id.string)	hash = scrape.mkhash(url+id.string)
links = []	links = []
for atag in row.find_all("a"):	for atag in row.find_all("a"):
if atag.has_key('href'):	if atag.has_key('href'):
links.append(scrape.fullurl(url,atag['href']))	links.append(scrape.fullurl(url,atag['href']))
doc = foidocsdb.get(hash)	doc = foidocsdb.get(hash)
descriptiontxt = ""	descriptiontxt = ""
for string in description.stripped_strings:	for string in description.stripped_strings:
descriptiontxt = descriptiontxt + " \n" + string	descriptiontxt = descriptiontxt + " \n" + string

if doc == None:	if doc == None:
print "saving"	print "saving"
dtresult = cal.parseDateText(date.string)	dtresult = cal.parseDateText(date.string)
if len(dtresult) == 2:	if len(dtresult) == 2:
(dtdate,dtr) = dtresult	(dtdate,dtr) = dtresult
print dtdate	print dtdate
edate = ""+str(dtdate[0])+'-'+str(dtdate[1])+'-'+str(dtdate[2])	edate = ""+str(dtdate[0])+'-'+str(dtdate[1])+'-'+str(dtdate[2])
else:	else:
edate = ""	edate = ""
doc = {'id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), "links": links, 'docID': id.string,	doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), "links": links, 'docID': id.string,
"date": edate, "description": descriptiontxt,"title": title.string,"notes": notes.string}	"date": edate, "description": descriptiontxt,"title": title.string,"notes": notes.string}
foidocsdb.save(doc)	foidocsdb.save(doc)
else:	else:
print "already saved"	print "already saved"

elif len(row.find_all('th')) == 5:	elif len(row.find_all('th')) == 5:
print "header row"	print "header row"

else:	else:
print "ERROR number of columns incorrect"	print "ERROR number of columns incorrect"
print row	print row

file:a/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py -> file:b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py

import sys,os	import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers	import genericScrapers
#RSS feed not detailed	#RSS feed not detailed
	import scrape
	from bs4 import BeautifulSoup
#http://www.doughellmann.com/PyMOTW/abc/	#http://www.doughellmann.com/PyMOTW/abc/
class ScraperImplementation(genericScrapers.GenericRSSDisclogScraper):	class ScraperImplementation(genericScrapers.GenericRSSDisclogScraper):
def getAgencyID(self):	def getAgencyID(self):
return "be9996f0ac58f71f23d074e82d44ead3"	return "be9996f0ac58f71f23d074e82d44ead3"

def getURL(self):	def getURL(self):
return "http://foi.deewr.gov.au/disclosure-log/rss"	return "http://foi.deewr.gov.au/disclosure-log/rss"

	def getDescription(self,entry,doc):
	(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, entry.link, "foidocuments", self.getAgencyID(), False)
	if content != None:
	if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
	# http://www.crummy.com/software/BeautifulSoup/documentation.html
	soup = BeautifulSoup(content)
	links = []
	description = ""
	dldivs = soup.find('div',class_="download")
	if dldivs != None:
	for atag in dldivs.find_all("a"):
	if atag.has_key('href'):
	links.append(scrape.fullurl(url,atag['href']))
	nodldivs = soup.find('div',class_="incompleteNotification")
	if nodldivs != None and nodldivs.stripped_strings != None:
	for text in nodldivs.stripped_strings:
	description = description + text
	for row in soup.table.find_all('tr'):
	if row != None:
	description = description + "\n" + row.find('th').string + ": "
	for text in row.find('div').stripped_strings:
	description = description + text
	if links != []:
	doc.update({'links': links})
	if description != "":
	doc.update({ 'description': description})


if __name__ == '__main__':	if __name__ == '__main__':
print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericRSSDisclogScraper)	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericRSSDisclogScraper)
print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericRSSDisclogScraper)	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericRSSDisclogScraper)
ScraperImplementation().doScrape()	ScraperImplementation().doScrape()

file:a/documents/template.inc.php -> file:b/documents/template.inc.php

<?php	<?php

function include_header_documents($title) {	function include_header_documents($title) {
?>	?>
<!doctype html>	<!doctype html>
<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->	<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->
<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->	<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->	<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->	<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->	<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->	<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
<head>	<head>
<meta charset="utf-8">	<meta charset="utf-8">

<!-- Use the .htaccess and remove these lines to avoid edge case issues.	<!-- Use the .htaccess and remove these lines to avoid edge case issues.
More info: h5bp.com/i/378 -->	More info: h5bp.com/i/378 -->
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">	<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">

<title>Australian Disclosure Logs<?php if ($title != "") echo " - $title";?></title>	<title>Australian Disclosure Logs<?php if ($title != "") echo " - $title";?></title>
<meta name="description" content="">	<meta name="description" content="">

<!-- Mobile viewport optimized: h5bp.com/viewport -->	<!-- Mobile viewport optimized: h5bp.com/viewport -->
<meta name="viewport" content="width=device-width">	<meta name="viewport" content="width=device-width">

<!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->	<!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />	<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />

<!-- Le styles -->	<!-- Le styles -->
<link href="css/bootstrap.min.css" rel="stylesheet">	<link href="css/bootstrap.min.css" rel="stylesheet">
<style type="text/css">	<style type="text/css">
body {	body {
padding-top: 60px;	padding-top: 60px;
padding-bottom: 40px;	padding-bottom: 40px;
}	}
.sidebar-nav {	.sidebar-nav {
padding: 9px 0;	padding: 9px 0;
}	}
</style>	</style>
<link href="css/bootstrap-responsive.min.css" rel="stylesheet">	<link href="css/bootstrap-responsive.min.css" rel="stylesheet">

<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->	<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
<!--[if lt IE 9]>	<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>	<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->	<![endif]-->
<!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->	<!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->

<!-- All JavaScript at the bottom, except this Modernizr build.	<!-- All JavaScript at the bottom, except this Modernizr build.
Modernizr enables HTML5 elements & feature detects for optimal performance.	Modernizr enables HTML5 elements & feature detects for optimal performance.
Create your own custom Modernizr build: www.modernizr.com/download/ -->	Create your own custom Modernizr build: www.modernizr.com/download/ -->
<script src="js/libs/modernizr-2.5.3.min.js"></script>	<script src="js/libs/modernizr-2.5.3.min.js"></script>

</head>	</head>
<body>	<body>
<div class="navbar navbar-inverse navbar-fixed-top">	<div class="navbar navbar-inverse navbar-fixed-top">
<div class="navbar-inner">	<div class="navbar-inner">
<div class="container-fluid">	<div class="container-fluid">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">	<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>	<span class="icon-bar"></span>
<span class="icon-bar"></span>	<span class="icon-bar"></span>
<span class="icon-bar"></span>	<span class="icon-bar"></span>
</a>	</a>
<a class="brand" href="#">Australian Disclosure Logs</a>	<a class="brand" href="#">Australian Disclosure Logs</a>
<div class="nav-collapse collapse">	<div class="nav-collapse collapse">
<p class="navbar-text pull-right">	<p class="navbar-text pull-right">
Check out our subsites on:	Check out our subsites on:
<a href="http://orgs.disclosurelo.gs">Government Agencies</a>	<a href="http://orgs.disclosurelo.gs">Government Agencies</a>
• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>	• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>
• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>	• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>

</p>	</p>
<ul class="nav">	<ul class="nav">
<li class="active"><a href="#">Home</a></li>	<li><a href="index.php">Home</a></li>
<li><a href="disclogsList.php">List of Disclosure Logs</a></li>	<li><a href="disclogsList.php">List of Disclosure Logs</a></li>
<li><a href="about.php">About</a></li>	<li><a href="about.php">About</a></li>

</ul>	</ul>
</div><!--/.nav-collapse -->	</div><!--/.nav-collapse -->
</div>	</div>
</div>	</div>
</div>	</div>
<div class="container">	<div class="container">
<?php	<?php
}	}
function include_footer_documents() {	function include_footer_documents() {
?>	?>
</div> <!-- /container -->	</div> <!-- /container -->
<hr>	<hr>

<footer>	<footer>
<p>© Company 2012</p>	<p>© Company 2012</p>
</footer>	</footer>
<script type="text/javascript">	<script type="text/javascript">

var _gaq = _gaq \|\| [];	var _gaq = _gaq \|\| [];
_gaq.push(['_setAccount', 'UA-12341040-4']);	_gaq.push(['_setAccount', 'UA-12341040-4']);
_gaq.push(['_setDomainName', 'disclosurelo.gs']);	_gaq.push(['_setDomainName', 'disclosurelo.gs']);
_gaq.push(['_setAllowLinker', true]);	_gaq.push(['_setAllowLinker', true]);
_gaq.push(['_trackPageview']);	_gaq.push(['_trackPageview']);

(function() {	(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;	var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';	ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);	var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();	})();

</script>	</script>
<!-- Le javascript	<!-- Le javascript
================================================== -->	================================================== -->
<!-- Placed at the end of the document so the pages load faster	<!-- Placed at the end of the document so the pages load faster
<script src="js/jquery.js"></script>	<script src="js/jquery.js"></script>
<script src="js/bootstrap-transition.js"></script>	<script src="js/bootstrap-transition.js"></script>
<script src="js/bootstrap-alert.js"></script>	<script src="js/bootstrap-alert.js"></script>
<script src="js/bootstrap-modal.js"></script>	<script src="js/bootstrap-modal.js"></script>
<script src="js/bootstrap-dropdown.js"></script>	<script src="js/bootstrap-dropdown.js"></script>
<script src="js/bootstrap-scrollspy.js"></script>	<script src="js/bootstrap-scrollspy.js"></script>
<script src="js/bootstrap-tab.js"></script>	<script src="js/bootstrap-tab.js"></script>
<script src="js/bootstrap-tooltip.js"></script>	<script src="js/bootstrap-tooltip.js"></script>
<script src="js/bootstrap-popover.js"></script>	<script src="js/bootstrap-popover.js"></script>
<script src="js/bootstrap-button.js"></script>	<script src="js/bootstrap-button.js"></script>
<script src="js/bootstrap-collapse.js"></script>	<script src="js/bootstrap-collapse.js"></script>
<script src="js/bootstrap-carousel.js"></script>	<script src="js/bootstrap-carousel.js"></script>
<script src="js/bootstrap-typeahead.js"></script>-->	<script src="js/bootstrap-typeahead.js"></script>-->

</body>	</body>
</html>	</html>
<?php	<?php
}	}

function displayLogEntry($row, $idtoname) {	function displayLogEntry($row, $idtoname) {
echo "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".$row->value->description;	echo "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".$row->value->description;
if (isset($row->value->notes)) {	if (isset($row->value->notes)) {
echo " <br>Note: ".$row->value->notes;	echo " <br>Note: ".$row->value->notes;
}	}
echo "</p>";	echo "</p>";

if (isset($row->value->links)){	if (isset($row->value->links)){
echo "<h3>Links/Documents</h3><ul>";	echo "<h3>Links/Documents</h3><ul>";
foreach ($row->value->links as $link) {	foreach ($row->value->links as $link) {
echo "<li><a href='$link'>".$link."</a></li>";	echo "<li><a href='$link'>".$link."</a></li>";
}	}

echo "</ul>";	echo "</ul>";
}	}
echo "<small><A href='".$row->value->url."'>View original source...</a> ID: ".$row->value->docID."</small>";	echo "<small><A href='".$row->value->url."'>View original source...</a> ID: ".$row->value->docID."</small>";
echo"</div>";	echo"</div>";
}	}