From: Maxious Date: Tue, 16 Jul 2013 06:19:03 +0000 Subject: exclude document viewer X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=33152ec49ae33d83419fcebedbe0a413678dff55 --- exclude document viewer Former-commit-id: 3f93fcc7e6c4209f6f2b628c2e031cccff7fd3b9 --- --- a/admin/logo.svg +++ b/admin/logo.svg @@ -1,34 +1,34 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + --- a/admin/refreshDesignDoc.php +++ b/admin/refreshDesignDoc.php @@ -8,21 +8,13 @@ $obj->_id = "_design/" . urlencode("app"); $obj->language = "javascript"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; -$obj->views->byDate->map = "function(doc) { if (doc.title != "Disclosure Log Updated") { emit(doc.date, doc); } };"; +$obj->views->byDate->map = "function(doc) { if (doc.title != \"Disclosure Log Updated\") { emit(doc.date, doc); } };"; $obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };"; $obj->views->byDateMonthYear->reduce = "_count"; $obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };"; $obj->views->byAgencyID->reduce = "_count"; -$obj->views->fieldNames->map = ' -function(doc) { -for(var propName in doc) { - emit(propName, doc._id); - } - -}'; -$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { - return values.length; -}'; +$obj->views->fieldNames->map = 'function(doc) { for(var propName in doc) { emit(propName, doc._id); }}'; +$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { return values.length; }'; // allow safe updates (even if slightly slower due to extra: rev-detection check). $foidb->save($obj, true); --- a/documents/robots.txt +++ b/documents/robots.txt @@ -3,4 +3,5 @@ User-agent: * Disallow: /admin/ +Disallow: /viewDocument.php Sitemap: http://disclosurelo.gs/sitemap.xml.php --- a/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py +++ b/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py @@ -7,12 +7,12 @@ #http://www.doughellmann.com/PyMOTW/abc/ class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): def getTable(self,soup): - return soup.find(id = "block-system-main").table + return soup.find(id = "ctl00_PlaceHolderMain_PublishingPageContent__ControlWrapper_RichHtmlField").table def getColumnCount(self): - return 2 + return 7 def getColumns(self,columns): - (date, title) = columns - return (date, date, title, title, None) + (id, date, title, description,link,deldate, notes) = columns + return (id, date, title, description, notes) if __name__ == '__main__': print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) --- a/documents/sitemap.xml.php +++ b/documents/sitemap.xml.php @@ -1,48 +1,30 @@ "; echo '' . "\n"; echo " " . local_url() . "index.php1.0\n"; foreach (scandir("./") as $file) { - if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php") { + if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php"&& $file != "viewDocument.php") echo " " . local_url() . "$file0.6\n"; - } } $agenciesdb = $server->get_db('disclosr-agencies'); -$foidocsdb = $server->get_db('disclosr-foidocuments'); try { $rows = $agenciesdb->get_view("app", "byCanonicalName")->rows; foreach ($rows as $row) { echo '' . local_url() . 'agency.php?id=' . $row->value->_id . "0.3\n"; } - unset($rows); - $rows = null; } catch (SetteeRestClientException $e) { setteErrorHandler($e); } - -foreach (range(0, 8) as $number) { +$foidocsdb = $server->get_db('disclosr-foidocuments'); try { - $rows = $foidocsdb->get_view("app", "all", Array($number,$number+1))->rows; + $rows = $foidocsdb->get_view("app", "all")->rows; foreach ($rows as $row) { echo '' . local_url() . 'view.php?id=' . $row->value->_id . "0.3\n"; } - unset($rows); - $rows = null; -} catch (SetteeRestClientException $e) { - setteErrorHandler($e); -} -} - -try { - $rows = $foidocsdb->get_view("app", "all", Array('9','fffffffff'))->rows; - foreach ($rows as $row) { - echo '' . local_url() . 'view.php?id=' . $row->value->_id . "0.3\n"; - } - unset($rows); - $rows = null; } catch (SetteeRestClientException $e) { setteErrorHandler($e); } --- /dev/null +++ b/exportAgencies.csv.php @@ -1,1 +1,88 @@ +get_db('disclosr-agencies'); + + +$foiEmail = Array(); +try { + $rows = $db->get_view("app", "foiEmails", null, true)->rows; + //print_r($rows); + foreach ($rows as $row) { + $foiEmail[$row->key] = $row->value; + } +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); + die(); +} + +$fp = fopen('php://output', 'w'); +if ($fp && $db) { + header('Content-Type: text/csv; charset=utf-8'); + header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); + header('Pragma: no-cache'); + header('Expires: 0'); + fputcsv($fp, $headers); + try { + $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; + //print_r($rows); + foreach ($agencies as $agency) { + // print_r($agency); + + if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) { + $row = Array(); + $row["title"] = trim($agency->value->name); + $row["abbr"] = (isset($agency->value->shortName) ? $agency->value->shortName : ""); + $row["key"] = (isset($agency->value->shortName) ? "au/".strtolower($agency->value->shortName) : ""); + $row["category"] =""; + $row["parent"] =""; + $row["parentkey"] =""; + $row["description"] = (isset($agency->value->description) ? $agency->value->description : ""); + $row["url"] = (isset($agency->value->website) ? $agency->value->website : ""); + $row["jurisdiction"] = "Australia"; + $row["jurisdiction_code"] = "au"; + + $row["source"] =""; + $row["source_url"] =""; + $row["address"] =""; + $row["contact"] =""; + + $row["email"] = (isset($agency->value->foiEmail) ? $agency->value->foiEmail : ""); + $row["tags"] =""; + $row["created_at"] =""; + $row["updated_at"] =""; + + + $otherBodies = Array(); + if (isset($agency->value->foiBodies)) { + $otherBodies = array_merge($otherBodies, $agency->value->foiBodies); + } + if (isset($agency->value->positions)) { + $positions = Array(); + foreach ($agency->value->positions as $position) { + $positions[] = "Office of the ".$position; + } + $otherBodies = array_merge($otherBodies, $positions); + } + sort($otherBodies); + if (count($otherBodies) > 0) { + $row["description"] .= "
This department also responds to requests for information held by " . implode(", ", $otherBodies); + } + + + fputcsv($fp, array_values($row)); + } + } + } catch (SetteeRestClientException $e) { + setteErrorHandler($e); + } + + die; +} +?> + --- a/getAgency.php +++ b/getAgency.php @@ -188,7 +188,7 @@
'; - echo '
'; + echo '
'; echo '

' . $row['name'] . '

'; if (isset($row['description'])) { --- a/include/common.inc.php +++ b/include/common.inc.php @@ -10,7 +10,6 @@ || strstr($_SERVER['PHP_SELF'], "documents/") || $_SERVER['SERVER_NAME'] == "disclosurelo.gs" || $_SERVER['SERVER_NAME'] == "www.disclosurelo.gs" - || $_SERVER['SERVER_NAME'] == "direct.disclosurelo.gs" ) $basePath = "../"; --- a/include/template.inc.php +++ b/include/template.inc.php @@ -44,7 +44,7 @@ - +
--- a/robots.txt +++ b/robots.txt @@ -3,4 +3,5 @@ User-agent: * Disallow: /admin/ +Disallow: /viewDocument.php Sitemap: http://orgs.disclosurelo.gs/sitemap.xml.php --- a/schemas/agency.json.php +++ b/schemas/agency.json.php @@ -4,7 +4,7 @@ "description" => "Representation of government agency and online transparency measures", "type" => "object", "properties" => Array( - "name" => Array("type" => "string", "required" => true, "x-itemprop" => "name", "x-property" => "schema:name foaf:name", "x-title" => "Name", "description" => "Name, most recent and broadest"), + "name" => Array("type" => "string", "required" => true, "x-itemprop" => "name", "x-property" => "schema:name foaf:name skos:prefLabel ", "x-title" => "Name", "description" => "Name, most recent and broadest"), "shortName" => Array("type" => "string", "required" => false, "x-title" => "Short Name", "description" => "Name shortened, usually to an acronym"), "description" => Array("type" => "string", "required" => false, "x-title" => "Description", "description" => "Description of roles and responsiblities of organisation"), "foiEmail" => Array("type" => "string", "required" => false, "x-title" => "FOI Contact Email", "x-itemprop" => "email", "description" => "FOI contact email if not foi@"), @@ -14,15 +14,15 @@ "items" => Array("type" => "string")), "positions" => Array("type" => "array", "required" => true, "x-title" => "Political Positions", "description" => "Ministers and Parliamentary Secretaries", "items" => Array("type" => "string")), - "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies","x-property"=>"schema:members foaf:knows", "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous", + "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies","x-property"=>"schema:members foaf:knows org:hasSubOrganization", "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous", "items" => Array("type" => "string")), "legislation" => Array("type" => "array", "required" => true, "x-title" => "Legislation", "description" => "Legislation administered by or created for the establishment of this organisation", "items" => Array("type" => "string")), - "orgType" => Array("type" => "string", "required" => true, "x-title" => "Organisation Type", "description" => "Org type based on legal formation via FMA/CAC legislation etc."), - "parentOrg" => Array("type" => "string", "required" => true, "x-title" => "Parent Organisation", "description" => "Parent organisation, usually a department of state"), + "orgType" => Array("type" => "string", "required" => true, "x-title" => "Organisation Type", "x-property" => "org:classification", "description" => "Org type based on legal formation via FMA/CAC legislation etc."), + "parentOrg" => Array("type" => "string", "required" => true, "x-title" => "Parent Organisation", "x-property" => "org:subOrganizationOf", "description" => "Parent organisation, usually a department of state"), "website" => Array("type" => "string", "required" => true, "x-title" => "Website", "x-itemprop" => "url", "x-property" => "schema:url foaf:homepage", "description" => "Website URL"), - "abn" => Array("type" => "string", "required" => true, "x-title" => "Australian Business Number", "description" => "ABN from business register"), - "established" => Array("type" => "string", "required" => true, "x-title" => "Date established", "description" => "Date established"), + "abn" => Array("type" => "string", "required" => true, "x-title" => "Australian Business Number", "x-property" => "org:identifier", "description" => "ABN from business register"), + "established" => Array("type" => "string", "required" => true, "x-title" => "Date established", "x-property" => "schema:foundingDate", "description" => "Date established"), "employees" => Array("type" => "string", "required" => true, "x-title" => "2010-2011 employees", "description" => "2010-2011 employees"), "contractListURL" => Array("type" => "string", "required" => true, "x-title" => "Contract Listing", "description" => "Departmental and agency contracts, mandated by the Senate" ), "budgetURL" => Array("type" => "string", "required" => true,"x-title" => "Budget", "description" => "Portfolio Budget Statements and Portfolio Additional Estimates Statements"), --- a/sitemap.xml.php +++ b/sitemap.xml.php @@ -7,7 +7,7 @@ echo '' . "\n"; echo " " . local_url() . "index.php1.0\n"; foreach (scandir("./") as $file) { - if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php") + if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php"&& $file != "viewDocument.php") echo " " . local_url() . "$file0.3\n"; }