ausaid scraper
ausaid scraper


Former-commit-id: d2ca4331376808c570b588bad9a3c97df495d510

<?php <?php
   
include('template.inc.php'); include('template.inc.php');
include_header_documents(""); include_header_documents("");
include_once('../include/common.inc.php'); include_once('../include/common.inc.php');
   
echo "<table> echo "<table>
<tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>"; <tr><th>Agency Name</th><th>Disclosure Log URL recorded?</th><th>Do we monitor this URL?</th></tr>";
$agenciesdb = $server->get_db('disclosr-agencies'); $agenciesdb = $server->get_db('disclosr-agencies');
$docsdb = $server->get_db('disclosr-documents'); $docsdb = $server->get_db('disclosr-documents');
try { try {
$rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows; $rows = $agenciesdb->get_view("app", "byCanonicalName", null, true)->rows;
   
   
if ($rows) { if ($rows) {
foreach ($rows as $row) { foreach ($rows as $row) {
   
echo "<tr><td><b>" . $row->value->name . "</b> (".$row->id.")</td>\n"; echo "<tr><td><b>" . $row->value->name . "</b>";
  if ($ENV == "DEV")
echo "<td>"; echo "<br>(" . $row->id . ")";
if (isset($row->value->FOIDocumentsURL)) { echo "</td>\n";
echo '<a href="viewDocument.php?hash='.md5($row->value->FOIDocumentsURL).'">'  
.$row->value->FOIDocumentsURL.'</a>';  
} else { echo "<td>";
  if (isset($row->value->FOIDocumentsURL)) {
  echo '<a href="' . $row->value->FOIDocumentsURL . '">'
  . $row->value->FOIDocumentsURL . '</a>';
  if ($ENV == "DEV")
  echo '<br><small>(<a href="viewDocument.php?hash=' . md5($row->value->FOIDocumentsURL) . '">'
  . 'view local copy</a>)</small>';
  } else {
echo "<font color='red'>✘</font>"; echo "<font color='red'>✘</font>";
} }
echo "</td>\n<td>"; echo "</td>\n<td>";
if (isset($row->value->FOIDocumentsURL)) { if (isset($row->value->FOIDocumentsURL)) {
if (file_exists("./scrapers/".$row->id.'.py')) { if (file_exists("./scrapers/" . $row->id . '.py')) {
echo "<font color='green'>✔</font>"; echo "<font color='green'>✔</font>";
} else if (file_exists("./scrapers/".$row->id.'.txt')){ } else if (file_exists("./scrapers/" . $row->id . '.txt')) {
echo "<font color='blue'><b>▬</b></font>"; echo "<font color='blue'><b>▬</b></font>";
} else { } else {
echo "<font color='red'>✘</font>"; echo "<font color='red'>✘</font>";
} }
} }
echo "</td></tr>\n"; echo "</td></tr>\n";
} }
} }
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
echo "</table>"; echo "</table>";
include_footer_documents(); include_footer_documents();
?> ?>
import sys,os import sys,os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import scrape import scrape
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import parsedatetime as pdt import parsedatetime as pdt
from time import mktime from time import mktime
from datetime import datetime from datetime import datetime
import feedparser import feedparser
import abc import abc
import unicodedata, re import unicodedata, re
   
class GenericDisclogScraper(object): class GenericDisclogScraper(object):
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
agencyID = None agencyID = None
disclogURL = None disclogURL = None
def remove_control_chars(self, input): def remove_control_chars(self, input):
return "".join([i for i in input if ord(i) in range(32, 127)]) return "".join([i for i in input if ord(i) in range(32, 127)])
def getAgencyID(self): def getAgencyID(self):
""" disclosr agency id """ """ disclosr agency id """
if self.agencyID == None: if self.agencyID == None:
self.agencyID = os.path.basename(sys.argv[0]).replace(".py","") self.agencyID = os.path.basename(sys.argv[0]).replace(".py","")
return self.agencyID return self.agencyID
   
def getURL(self): def getURL(self):
""" disclog URL""" """ disclog URL"""
if self.disclogURL == None: if self.disclogURL == None:
agency = scrape.agencydb.get(self.getAgencyID()) agency = scrape.agencydb.get(self.getAgencyID())
self.disclogURL = agency['FOIDocumentsURL'] self.disclogURL = agency['FOIDocumentsURL']
return self.disclogURL return self.disclogURL
   
@abc.abstractmethod @abc.abstractmethod
def doScrape(self): def doScrape(self):
""" do the scraping """ """ do the scraping """
return return
   
@abc.abstractmethod @abc.abstractmethod
def getDescription(self, content, entry, doc): def getDescription(self, content, entry, doc):
""" get description""" """ get description"""
return return
   
   
   
class GenericRSSDisclogScraper(GenericDisclogScraper): class GenericRSSDisclogScraper(GenericDisclogScraper):
   
def doScrape(self): def doScrape(self):
foidocsdb = scrape.couch['disclosr-foidocuments'] foidocsdb = scrape.couch['disclosr-foidocuments']
(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID()) (url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
feed = feedparser.parse(content) feed = feedparser.parse(content)
for entry in feed.entries: for entry in feed.entries:
#print entry #print entry
print entry.id print entry.id
hash = scrape.mkhash(entry.id) hash = scrape.mkhash(entry.id)
#print hash #print hash
doc = foidocsdb.get(hash) doc = foidocsdb.get(hash)
#print doc #print doc
if doc == None: if doc == None:
print "saving" print "saving"
edate = datetime.fromtimestamp(mktime( entry.published_parsed)).strftime("%Y-%m-%d") edate = datetime.fromtimestamp(mktime( entry.published_parsed)).strftime("%Y-%m-%d")
doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': entry.link, 'docID': entry.id, doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': entry.link, 'docID': entry.id,
"date": edate,"title": entry.title} "date": edate,"title": entry.title}
self.getDescription(entry,entry, doc) self.getDescription(entry,entry, doc)
foidocsdb.save(doc) foidocsdb.save(doc)
else: else:
print "already saved" print "already saved"
def getDescription(self, content, entry, doc): def getDescription(self, content, entry, doc):
""" get description from rss entry""" """ get description from rss entry"""
doc.update({'description': content.summary}) doc.update({'description': content.summary})
return return
   
class GenericOAICDisclogScraper(GenericDisclogScraper): class GenericOAICDisclogScraper(GenericDisclogScraper):
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
@abc.abstractmethod @abc.abstractmethod
def getColumns(self,columns): def getColumns(self,columns):
""" rearranges columns if required """ """ rearranges columns if required """
return return
def getColumnCount(self): def getColumnCount(self):
return 5 return 5
def getDescription(self, content, entry, doc): def getDescription(self, content, entry, doc):
""" get description from rss entry""" """ get description from rss entry"""
descriptiontxt = "" descriptiontxt = ""
for string in content.stripped_strings: for string in content.stripped_strings:
descriptiontxt = descriptiontxt + " \n" + string descriptiontxt = descriptiontxt + " \n" + string
doc.update({'description': descriptiontxt}) doc.update({'description': descriptiontxt})
return return
  def getTitle(self, content, entry, doc):
  doc.update({'title': content.string})
  return
def getTable(self, soup): def getTable(self, soup):
return soup.table return soup.table
  def getDate(self, content, entry, doc):
  dtresult = cal.parseDateText(content.string)
  if len(dtresult) == 2:
  (dtdate,dtr) = dtresult
  edate = ""+str(dtdate[0])+'-'+str(dtdate[1])+'-'+str(dtdate[2])
  else:
  edate = datetime.strptime(date.string.strip(), "%d %B %Y").strftime("%Y-%m-%d")
  print edate
  doc.update({'date': edate})
  return
   
def doScrape(self): def doScrape(self):
cal = pdt.Calendar() cal = pdt.Calendar()
foidocsdb = scrape.couch['disclosr-foidocuments'] foidocsdb = scrape.couch['disclosr-foidocuments']
(url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID()) (url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
if content != None: if content != None:
if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml":
# http://www.crummy.com/software/BeautifulSoup/documentation.html # http://www.crummy.com/software/BeautifulSoup/documentation.html
soup = BeautifulSoup(content) soup = BeautifulSoup(content)
table = self.getTable(soup) table = self.getTable(soup)
for row in table.find_all('tr'): for row in table.find_all('tr'):
columns = row.find_all('td') columns = row.find_all('td')
if len(columns) == self.getColumnCount(): if len(columns) == self.getColumnCount():
(id, date, description, title, notes) = self.getColumns(columns) (id, date, description, title, notes) = self.getColumns(columns)
print id.string print id.string
if id.string == None: if id.string == None:
hash = scrape.mkhash(self.remove_control_chars(url+date.string)) hash = scrape.mkhash(self.remove_control_chars(url+date.string))
else: else:
hash = scrape.mkhash(self.remove_control_chars(url+id.string)) hash = scrape.mkhash(self.remove_control_chars(url+id.string))
links = [] links = []
for atag in row.find_all("a"): for atag in row.find_all("a"):
if atag.has_key('href'): if atag.has_key('href'):
links.append(scrape.fullurl(url,atag['href'])) links.append(scrape.fullurl(url,atag['href']))
doc = foidocsdb.get(hash) doc = foidocsdb.get(hash)
if doc == None: if doc == None:
print "saving" print "saving"
dtresult = cal.parseDateText(date.string) doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), 'docID': id.string}
if len(dtresult) == 2:  
(dtdate,dtr) = dtresult  
print dtdate  
edate = ""+str(dtdate[0])+'-'+str(dtdate[1])+'-'+str(dtdate[2])  
else:  
edate = datetime.strptime(date.string.strip(), "%d %B %Y").strftime("%Y-%m-%d")  
doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), 'docID': id.string,  
"date": edate,"title": title.string}  
if links != []: if links != []:
doc.update({'links': links}) doc.update({'links': links})
  self.getTitle(title,row, doc)
  self.getDate(date,row, doc)
self.getDescription(description,row, doc) self.getDescription(description,row, doc)
if notes != None: if notes != None:
doc.update({ 'notes': notes.string}) doc.update({ 'notes': notes.string})
foidocsdb.save(doc) foidocsdb.save(doc)
else: else:
print "already saved "+hash print "already saved "+hash
elif len(row.find_all('th')) == self.getColumnCount(): elif len(row.find_all('th')) == self.getColumnCount():
print "header row" print "header row"
else: else:
print "ERROR number of columns incorrect" print "ERROR number of columns incorrect"
print row print row
   
<?php <?php
   
include_once('include/common.inc.php'); include_once('include/common.inc.php');
   
function displayValue($key, $value, $mode) { function displayValue($key, $value, $mode) {
global $db, $schemas; global $db, $schemas;
if ($mode == "view") { if ($mode == "view") {
if (strpos($key, "_") === 0 || $key == "metadata") if (strpos($key, "_") === 0 || $key == "metadata")
return; return;
echo "<tr>"; echo "<tr>";
   
echo "<td>"; echo "<td>";
if (isset($schemas['agency']["properties"][$key])) { if (isset($schemas['agency']["properties"][$key])) {
echo $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small>"; echo $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small>";
} }
echo "</td><td>"; echo "</td><td>";
if (is_array($value)) { if (is_array($value)) {
echo "<ol>"; echo "<ol>";
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
   
echo "<li "; echo "<li ";
if (isset($schemas['agency']["properties"][$key]['x-property'])) { if (isset($schemas['agency']["properties"][$key]['x-property'])) {
echo ' property="' . $schemas['agency']["properties"][$key]['x-property'] . '" '; echo ' property="' . $schemas['agency']["properties"][$key]['x-property'] . '" ';
} if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) { } if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
echo ' itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '" '; echo ' itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '" ';
} }
echo " >"; echo " >";
   
echo "$subvalue</li>"; echo "$subvalue</li>";
} }
echo "</ol></td></tr>"; echo "</ol></td></tr>";
} else { } else {
if (isset($schemas['agency']["properties"][$key]['x-property'])) { if (isset($schemas['agency']["properties"][$key]['x-property'])) {
echo '<span property="' . $schemas['agency']["properties"][$key]['x-property'] . '">'; echo '<span property="' . $schemas['agency']["properties"][$key]['x-property'] . '">';
} else { } else {
echo "<span>"; echo "<span>";
} }
   
if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") { if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
echo "<a " . ($key == 'website' ? 'itemprop="url"' : '') . " href='$value'>$value</a>"; echo "<a " . ($key == 'website' ? 'itemprop="url"' : '') . " href='$value'>$value</a>";
} else if ($key == 'abn') { } else if ($key == 'abn') {
echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>$value</a>"; echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>$value</a>";
} else { } else {
echo "$value"; echo "$value";
} }
echo "</span>"; echo "</span>";
} }
echo "</td></tr>"; echo "</td></tr>";
} }
if ($mode == "edit") { if ($mode == "edit") {
if (is_array($value)) { if (is_array($value)) {
echo '<div class="row"> echo '<div class="row">
<div class="seven columns"> <div class="seven columns">
<fieldset> <fieldset>
<h5>' . $key . '</h5>'; <h5>' . $key . '</h5>';
foreach ($value as $subkey => $subvalue) { foreach ($value as $subkey => $subvalue) {
echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>"; echo "<label>$subkey</label><input class='input-text' type='text' id='$key$subkey' name='$key" . '[' . $subkey . "]' value='$subvalue'/></tr>";
} }
echo "</fieldset> echo "</fieldset>
</div> </div>
</div>"; </div>";
} else { } else {
if (strpos($key, "_") === 0) { if (strpos($key, "_") === 0) {
echo"<input type='hidden' id='$key' name='$key' value='$value'/>"; echo"<input type='hidden' id='$key' name='$key' value='$value'/>";
} else if ($key == "parentOrg") { } else if ($key == "parentOrg") {
echo "<label for='$key'>$key</label><select id='$key' name='$key'><option value=''> Select... </option>"; echo "<label for='$key'>$key</label><select id='$key' name='$key'><option value=''> Select... </option>";
$rows = $db->get_view("app", "byDeptStateName")->rows; $rows = $db->get_view("app", "byDeptStateName")->rows;
//print_r($rows); //print_r($rows);
foreach ($rows as $row) { foreach ($rows as $row) {
echo "<option value='{$row->value}'" . (($row->value == $value) ? "SELECTED" : "") . " >" . str_replace("Department of ", "", $row->key) . "</option>"; echo "<option value='{$row->value}'" . (($row->value == $value) ? "SELECTED" : "") . " >" . str_replace("Department of ", "", $row->key) . "</option>";
} }
echo" </select>"; echo" </select>";
} else { } else {
echo "<label>$key</label><input class='input-text' type='text' id='$key' name='$key' value='$value'/>"; echo "<label>$key</label><input class='input-text' type='text' id='$key' name='$key' value='$value'/>";
if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") { if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
echo "<a " . ($key == 'website' ? 'itemprop="url"' : '') . " href='$value'>view</a>"; echo "<a " . ($key == 'website' ? 'itemprop="url"' : '') . " href='$value'>view</a>";
} }
if ($key == 'abn') { if ($key == 'abn') {
echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>"; echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>";
} }
} }
} }
} }
// //
} }
   
function addDefaultFields($row) { function addDefaultFields($row) {
global $schemas; global $schemas;
$defaultFields = array_keys($schemas['agency']['properties']); $defaultFields = array_keys($schemas['agency']['properties']);
foreach ($defaultFields as $defaultField) { foreach ($defaultFields as $defaultField) {
if (!isset($row[$defaultField])) { if (!isset($row[$defaultField])) {
if ($schemas['agency']['properties'][$defaultField]['type'] == "string") { if ($schemas['agency']['properties'][$defaultField]['type'] == "string") {
$row[$defaultField] = ""; $row[$defaultField] = "";
} }
if ($schemas['agency']['properties'][$defaultField]['type'] == "array") { if ($schemas['agency']['properties'][$defaultField]['type'] == "array") {
$row[$defaultField] = Array(""); $row[$defaultField] = Array("");
} }
} else if ($schemas['agency']['properties'][$defaultField]['type'] == "array") { } else if ($schemas['agency']['properties'][$defaultField]['type'] == "array") {
if (is_array($row[$defaultField])) { if (is_array($row[$defaultField])) {
$row[$defaultField][] = ""; $row[$defaultField][] = "";
$row[$defaultField][] = ""; $row[$defaultField][] = "";
$row[$defaultField][] = ""; $row[$defaultField][] = "";
} else { } else {
$value = $row[$defaultField]; $value = $row[$defaultField];
$row[$defaultField] = Array($value); $row[$defaultField] = Array($value);
$row[$defaultField][] = ""; $row[$defaultField][] = "";
$row[$defaultField][] = ""; $row[$defaultField][] = "";
} }
} }
} }
return $row; return $row;
} }
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
   
if (isset($_REQUEST['id'])) { if (isset($_REQUEST['id'])) {
//get an agency record as json/html, search by name/abn/id //get an agency record as json/html, search by name/abn/id
// by name = startkey="Ham"&endkey="Ham\ufff0" // by name = startkey="Ham"&endkey="Ham\ufff0"
// edit? // edit?
   
$obj = $db->get($_REQUEST['id']); $obj = $db->get($_REQUEST['id']);
include_header($obj->name); include_header(isset($obj->name) ? $obj->name : "");
//print_r($row); //print_r($row);
if (sizeof($_POST) > 0) { if (sizeof($_POST) > 0) {
//print_r($_POST); //print_r($_POST);
foreach ($_POST as $postkey => $postvalue) { foreach ($_POST as $postkey => $postvalue) {
if ($postvalue == "") { if ($postvalue == "") {
unset($_POST[$postkey]); unset($_POST[$postkey]);
} }
if (is_array($postvalue)) { if (is_array($postvalue)) {
if (count($postvalue) == 1 && $postvalue[0] == "") { if (count($postvalue) == 1 && $postvalue[0] == "") {
unset($_POST[$postkey]); unset($_POST[$postkey]);
} else { } else {
foreach ($_POST[$postkey] as $key => &$value) { foreach ($_POST[$postkey] as $key => &$value) {
if ($value == "") { if ($value == "") {
unset($_POST[$postkey][$key]); unset($_POST[$postkey][$key]);
} }
} }
} }
} }
} }
if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) { if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) {
echo "Edited version was latest version, continue saving"; echo "Edited version was latest version, continue saving";
$newdoc = $_POST; $newdoc = $_POST;
$newdoc['metadata']['lastModified'] = time(); $newdoc['metadata']['lastModified'] = time();
$obj = $db->save($newdoc); $obj = $db->save($newdoc);
} else { } else {
echo "ALERT doc revised by someone else while editing. Document not saved."; echo "ALERT doc revised by someone else while editing. Document not saved.";
} }
} }
   
$mode = "view"; $mode = "view";
$rowArray = object_to_array($obj); $rowArray = object_to_array($obj);
ksort($rowArray); ksort($rowArray);
if ($mode == "edit") { if ($mode == "edit") {
$row = addDefaultFields($rowArray); $row = addDefaultFields($rowArray);
} else { } else {
$row = $rowArray; $row = $rowArray;
} }
   
if ($mode == "view") { if ($mode == "view") {
echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization" about="#' . $row['_id'] . '"><table width="100%">'; echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization" about="#' . $row['_id'] . '"><table width="100%">';
echo '<tr> <td colspan="2"><h3 itemprop="name">' . $row['name'] . "</h3></td></tr>"; echo '<tr> <td colspan="2"><h3 itemprop="name">' . $row['name'] . "</h3></td></tr>";
echo "<tr><th>Field Name</th><th>Field Value</th></tr>"; echo "<tr><th>Field Name</th><th>Field Value</th></tr>";
} }
if ($mode == "edit") { if ($mode == "edit") {
?> ?>
<input id="addfield" type="button" value="Add Field"/> <input id="addfield" type="button" value="Add Field"/>
<script> <script>
window.onload = function() { window.onload = function() {
$(document).ready(function() { $(document).ready(function() {
// put all your jQuery goodness in here. // put all your jQuery goodness in here.
// http://charlie.griefer.com/blog/2009/09/17/jquery-dynamically-adding-form-elements/ // http://charlie.griefer.com/blog/2009/09/17/jquery-dynamically-adding-form-elements/
$('#addfield').click(function() { $('#addfield').click(function() {
var field_name=window.prompt("fieldname?",""); var field_name=window.prompt("fieldname?","");
if (field_name !="") { if (field_name !="") {
$('#submitbutton').before($('<span></span>') $('#submitbutton').before($('<span></span>')
.append("<label>"+field_name+"</label>") .append("<label>"+field_name+"</label>")
.append("<input class='input-text' type='text' id='"+field_name+"' name='"+field_name+"'/>") .append("<input class='input-text' type='text' id='"+field_name+"' name='"+field_name+"'/>")
); );
} }
}); });
}); });
}; };
</script> </script>
<form id="editform" class="nice" method="post"> <form id="editform" class="nice" method="post">
<?php <?php
   
} }
foreach ($row as $key => $value) { foreach ($row as $key => $value) {
echo displayValue($key, $value, $mode); echo displayValue($key, $value, $mode);
} }
if ($mode == "view") { if ($mode == "view") {
echo "</table></div>"; echo "</table></div>";
} }
if ($mode == "edit") { if ($mode == "edit") {
echo '<input id="submitbutton" type="submit"/></form>'; echo '<input id="submitbutton" type="submit"/></form>';
} }
} else { } else {
// show all list // show all list
include_header('Agencies'); include_header('Agencies');
try { try {
/* $rows = $db->get_view("app", "showNamesABNs")->rows;  
//print_r($rows);  
foreach ($rows as $row) {  
// print_r($row);  
echo '<li><a href="getAgency.php?id=' . $row->key . '">' .  
(isset($row->value->name) && $row->value->name != "" ? $row->value->name : "NO NAME " . $row->value->abn)  
. '</a></li>';  
} */  
$rows = $db->get_view("app", "byCanonicalName")->rows; $rows = $db->get_view("app", "byCanonicalName")->rows;
//print_r($rows); //print_r($rows);
echo '<ul>'; echo '<ul>';
foreach ($rows as $row) { foreach ($rows as $row) {
// print_r($row); // print_r($row);
echo '<li itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization foaf:Organization" about="getAgency.php?id=' . $row->value->_id . '"> echo '<li itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization foaf:Organization" about="getAgency.php?id=' . $row->value->_id . '">
<a href="getAgency.php?id=' . $row->value->_id . '" rel="schema:url foaf:page" property="schema:name foaf:name" itemprop="url"><span itemprop="name">' . <a href="getAgency.php?id=' . $row->value->_id . '" rel="schema:url foaf:page" property="schema:name foaf:name" itemprop="url"><span itemprop="name">' .
$row->value->name (isset($row->value->name) ? $row->value->name : "ERROR NAME MISSING")
. '</span></a></li>'; . '</span></a></li>';
} }
echo "</ul>"; echo "</ul>";
} catch (SetteeRestClientException $e) { } catch (SetteeRestClientException $e) {
setteErrorHandler($e); setteErrorHandler($e);
} }
} }
include_footer(); include_footer();
?> ?>
   
<?php <?php
   
date_default_timezone_set("Australia/Sydney"); date_default_timezone_set("Australia/Sydney");
   
$basePath = ""; $basePath = "";
if (strstr($_SERVER['PHP_SELF'], "alaveteli/") if (strstr($_SERVER['PHP_SELF'], "alaveteli/")
|| strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "admin/")
|| strstr($_SERVER['PHP_SELF'], "lib/") || strstr($_SERVER['PHP_SELF'], "lib/")
|| strstr($_SERVER['PHP_SELF'], "include/") || strstr($_SERVER['PHP_SELF'], "include/")
|| strstr($_SERVER['PHP_SELF'], "documents/") || strstr($_SERVER['PHP_SELF'], "documents/")
|| $_SERVER['SERVER_NAME'] == "disclosurelo.gs" || $_SERVER['SERVER_NAME'] == "disclosurelo.gs"
) )
$basePath = "../"; $basePath = "../";
   
include_once ('couchdb.inc.php'); include_once ('couchdb.inc.php');
include_once ('template.inc.php'); include_once ('template.inc.php');
require_once $basePath.'lib/Requests/library/Requests.php'; require_once $basePath.'lib/Requests/library/Requests.php';
   
Requests::register_autoloader(); Requests::register_autoloader();
  $ENV = "DEV";
if (isset($_SERVER['SERVER_NAME']) && $_SERVER['SERVER_NAME'] != 'localhost') { if (isset($_SERVER['SERVER_NAME']) && $_SERVER['SERVER_NAME'] != 'localhost') {
   
require $basePath."lib/amon-php/amon.php"; require $basePath."lib/amon-php/amon.php";
Amon::config(array('address'=> 'http://127.0.0.1:2464', Amon::config(array('address'=> 'http://127.0.0.1:2464',
'protocol' => 'http', 'protocol' => 'http',
'secret_key' => "I2LJ6dOMmlnXgVAkTPFXd5M3ejkga8Gd2FbBt6iqZdw")); 'secret_key' => "I2LJ6dOMmlnXgVAkTPFXd5M3ejkga8Gd2FbBt6iqZdw"));
Amon::setup_exception_handler(); Amon::setup_exception_handler();
  $ENV = "PROD";
} }
   
# Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735 # Convert a stdClass to an Array. http://www.php.net/manual/en/language.types.object.php#102735
   
function object_to_array(stdClass $Class) { function object_to_array(stdClass $Class) {
# Typecast to (array) automatically converts stdClass -> array. # Typecast to (array) automatically converts stdClass -> array.
$Class = (array) $Class; $Class = (array) $Class;
   
# Iterate through the former properties looking for any stdClass properties. # Iterate through the former properties looking for any stdClass properties.
# Recursively apply (array). # Recursively apply (array).
foreach ($Class as $key => $value) { foreach ($Class as $key => $value) {
if (is_object($value) && get_class($value) === 'stdClass') { if (is_object($value) && get_class($value) === 'stdClass') {
$Class[$key] = object_to_array($value); $Class[$key] = object_to_array($value);
} }
} }
return $Class; return $Class;
} }
   
# Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735 # Convert an Array to stdClass. http://www.php.net/manual/en/language.types.object.php#102735
   
function array_to_object(array $array) { function array_to_object(array $array) {
# Iterate through our array looking for array values. # Iterate through our array looking for array values.
# If found recurvisely call itself. # If found recurvisely call itself.
foreach ($array as $key => $value) { foreach ($array as $key => $value) {
if (is_array($value)) { if (is_array($value)) {
$array[$key] = array_to_object($value); $array[$key] = array_to_object($value);
} }
} }
   
# Typecast to (object) will automatically convert array -> stdClass # Typecast to (object) will automatically convert array -> stdClass
return (object) $array; return (object) $array;
} }
   
function dept_to_portfolio($deptName) { function dept_to_portfolio($deptName) {
return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName))); return trim(str_replace("Department of", "", str_replace("Department of the", "Department of", $deptName)));
} }
function phrase_to_tag ($phrase) { function phrase_to_tag ($phrase) {
return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase)))); return str_replace(" ","_",str_replace("'","",str_replace(",","",strtolower($phrase))));
} }
function local_url() { function local_url() {
return "http://" . $_SERVER['HTTP_HOST'] . rtrim(dirname($_SERVER['PHP_SELF']), '/\\') . "/"; return "http://" . $_SERVER['HTTP_HOST'] . rtrim(dirname($_SERVER['PHP_SELF']), '/\\') . "/";
} }
function GetDomain($url) function GetDomain($url)
{ {
$nowww = ereg_replace('www\.','',$url); $nowww = ereg_replace('www\.','',$url);
$domain = parse_url($nowww); $domain = parse_url($nowww);
if(!empty($domain["host"])) if(!empty($domain["host"]))
{ {
return $domain["host"]; return $domain["host"];
} else } else
{ {
return $domain["path"]; return $domain["path"];
} }
} }