gitphp 0.2.9.1 :: disclosr.git/commitdiff

more scrapers

Former-commit-id: b3ae23479c9f8a0c79c9181bdd1ce0f5b3ea0656

9 files changed: (show all)
documents/disclogsList.php
documents/scrapers/1097fa8afdcf5db89d212d0979226667.py (new)
documents/scrapers/1ad74ca88932f90f0b92b69387171441.py (new)
documents/scrapers/31b7c75cf484747b6b120680bddd33b0.py (new)
documents/scrapers/329fccdca068b78ab7edd550e2957398.py (new)
documents/scrapers/6fe3c812a99d486963133459b2768cf6.py (new)
documents/scrapers/734f12db07e844b30cd11dc98500f2ce.py (new)
documents/scrapers/955dc4cb047b5439dfb65549ce2696a6.py (new)
documents/scrapers/c414e65ed728d05307d5b27f13e195e1.py (new)

file:a/documents/disclogsList.php -> file:b/documents/disclogsList.php

<?php	<?php

include('template.inc.php');	include('template.inc.php');
include_header_documents("");	include_header_documents("");
include_once('../include/common.inc.php');	include_once('../include/common.inc.php');

echo "<table>	echo "<table>
<tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>";	<tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>";
$agenciesdb = $server->get_db('disclosr-agencies');	$agenciesdb = $server->get_db('disclosr-agencies');
$docsdb = $server->get_db('disclosr-documents');	$docsdb = $server->get_db('disclosr-documents');
try {	try {
$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;	$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;


if ($rows) {	if ($rows) {
foreach ($rows as $row) {	foreach ($rows as $row) {

echo "<tr><td><b>" . $row->value->name . "</b>";	echo "<tr><td><b>" . $row->value->name . "</b>";
if ($ENV == "DEV")	if ($ENV == "DEV")
echo "<br>(" . $row->id . ")";	echo "<br>(" . $row->id . ")";
echo "</td>\n";	echo "</td>\n";


echo "<td>";	echo "<td>";
if (isset($row->value->FOIDocumentsURL)) {	if (isset($row->value->FOIDocumentsURL)) {
echo '<a href="' . $row->value->FOIDocumentsURL . '">'	echo '<a href="' . $row->value->FOIDocumentsURL . '">'
. $row->value->FOIDocumentsURL . '</a>';	. $row->value->FOIDocumentsURL . '</a>';
if ($ENV == "DEV")	if ($ENV == "DEV")
echo '<br><small>(<a href="viewDocument.php?hash=' . md5($row->value->FOIDocumentsURL) . '">'	echo '<br><small>(<a href="viewDocument.php?hash=' . md5($row->value->FOIDocumentsURL) . '">'
. 'view local copy</a>)</small>';	. 'view local copy</a>)</small>';
} else {	} else {
echo "<font color='red'>✘</font>";	echo "<font color='red'>✘</font>";
}	}
echo "</td>\n<td>";	echo "</td>\n<td>";
if (isset($row->value->FOIDocumentsURL)) {	if (isset($row->value->FOIDocumentsURL)) {
if (file_exists("./scrapers/" . $row->id . '.py')) {	if (file_exists("./scrapers/" . $row->id . '.py')) {
echo "<font color='green'>✔</font>";	echo "<font color='green'>✔</font>";
} else if (file_exists("./scrapers/" . $row->id . '.txt')) {	} else if (file_exists("./scrapers/" . $row->id . '.txt')) {
echo "<font color='blue'><b>▬</b></font>";	echo "<font color='orange'><b>▬</b></font>";
} else {	} else {
echo "<font color='red'>✘</font>";	echo "<font color='red'>✘</font>";
}	}
}	}
echo "</td></tr>\n";	echo "</td></tr>\n";
}	}
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}
echo "</table>";	echo "</table>";
include_footer_documents();	include_footer_documents();
?>	?>

file:b/documents/scrapers/1097fa8afdcf5db89d212d0979226667.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumns(self,columns):
	(id, date, title, description, notes) = columns
	return (id, date, description, title, notes)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/1ad74ca88932f90f0b92b69387171441.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumns(self,columns):
	(id, date, title, description, notes) = columns
	return (id, date, title, description, notes)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/31b7c75cf484747b6b120680bddd33b0.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumnCount(self):
	return 2
	def getColumns(self,columns):
	(title, date) = columns
	return (date, date, title, title, None)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/329fccdca068b78ab7edd550e2957398.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumnCount(self):
	return 7
	def getTable(self,soup):
	return soup.find(class_ = "foi-disclosure")
	def getColumns(self,columns):
	(disclogid, id, date, title, link, removedate, notes) = columns
	return (id, date, title, title, notes)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/6fe3c812a99d486963133459b2768cf6.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumns(self,columns):
	(id, date, title, description, notes) = columns
	return (id, date, description, title, notes)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/734f12db07e844b30cd11dc98500f2ce.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumnCount(self):
	return 4
	def getColumns(self,columns):
	(id, date, title, description) = columns
	return (id, date, title, description, None)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/955dc4cb047b5439dfb65549ce2696a6.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumns(self,columns):
	(id, date, title, description, notes) = columns
	return (id, date, title, description, notes)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()

file:b/documents/scrapers/c414e65ed728d05307d5b27f13e195e1.py (new)

	import sys,os
	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
	import genericScrapers
	import scrape
	from bs4 import BeautifulSoup

	#http://www.doughellmann.com/PyMOTW/abc/
	class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
	def getColumnCount(self):
	return 3
	def getColumns(self,columns):
	(date, title, description) = columns
	return (date, date, title, description, None)

	if __name__ == '__main__':
	print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)
	print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper)
	ScraperImplementation().doScrape()