Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr

Conflicts:
documents/sitemap.xml.php

Former-commit-id: 0480844aebe20a3af30b1c98aa192a5a823fb4d6

--- a/admin/logo.svg
+++ b/admin/logo.svg
@@ -1,34 +1,34 @@
-<?xml version="1.0" encoding="utf-8"?>

-<!-- Generator: Adobe Illustrator 15.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->

-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">

-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="100px"

-	 height="100px" viewBox="0 -25.635 100 100" enable-background="new 0 -25.635 100 100" xml:space="preserve">

-<g id="docs">

-	<path fill="#C2A385" d="M86.108-9.909l5.229,9.952c0,0-1.832,0.083-5.297,1.95c-2.312,1.249-6.468,6.246-6.468,6.246L71.827-2.909

-		c0,0,4.201-3.996,6.513-5.242C81.805-10.022,86.108-9.909,86.108-9.909"/>

-	<path fill="#C2A385" d="M65.604,20.731l-3.152-9.868c0,0-5.441,3.56-8.017,4.074c-1.008,0.202-1.93,0.335-2.749,0.425

-		L65.604,20.731z"/>

-	<path fill="#C2A385" d="M72.326,23.321c0.268-0.226,0.537-0.44,0.804-0.616c3.104-2.054,6.139-3.685,6.268-3.755l1.882-1.005

-		l1.369,1.634l2.864,3.417l3.198-4.334L76.68,9.783l-8.74,11.847L72.326,23.321z"/>

-	<path fill="#C2A385" d="M39.918,10.823l4.825,1.86l3.33,0.212c0.04,0.001,0.269,0.015,0.652,0.015c0.91,0,2.798-0.072,5.196-0.551

-		c1.427-0.284,5.007-2.332,7.092-3.695l2.889-1.888l1.05,3.285l2.496,7.812l5.889-7.985l-4.625,0.163l1.348-6.225L55.133,0.593

-		l-2.095,9.667c-0.531-2.599-1.841-5.727-1.841-5.727L37.709,6.055c0,0,0.885,2.206,1.586,4.529L39.918,10.823z"/>

-	<path fill="#C2A385" d="M91.233,45.562c-1.102-0.691-2.323-1.142-3.415-1.433l-3.779,9.804c1.932,1.246,5.197,5.738,5.197,5.738

-		l7.336-9.206C96.572,50.466,93.162,46.771,91.233,45.562z"/>

-	<path fill="#C2A385" d="M93.192,32.166l-3.656,1.224c-0.019,0.007-1.779,0.613-4.117,2.069l2.817,4.868l0.626,1.08

-		c3.306-0.562,7.727-1.922,7.727-1.922l-2.332-15.261c0,0-2.934,1.277-5.852,2.221l2.318,2.765L93.192,32.166z"/>

-	<path fill="#C2A385" d="M79.272,25.999l0.864,0.334l0.46,0.801l3.503,6.05c2.646-1.636,4.611-2.287,4.611-2.287l-8.075-9.632

-		c0,0-2.584,1.391-5.376,3.188L79.272,25.999z"/>

-</g>

-<g id="trunk">

-	<circle fill="#C00000" cx="66.019" cy="52.945" r="7.877"/>

-	<circle fill="#C00000" cx="22.693" cy="52.945" r="7.877"/>

-	<path fill="#C00000" d="M22.693,42.441c1.915,0,3.705,0.522,5.251,1.421V28.001H12.191L3,37.192v14.439h9.281

-		C12.931,46.459,17.347,42.441,22.693,42.441z"/>

-	<path fill="#C00000" d="M55.925,50.075l-9.583-3.695H30.88c1.186,1.476,1.978,3.28,2.225,5.252h22.502

-		C55.674,51.1,55.78,50.58,55.925,50.075z"/>

-	<path fill="#C00000" d="M79.312,28.328L39.961,13.149l-9.384,24.335l26.381,10.174c1.824-3.115,5.198-5.218,9.062-5.218

-		c5.791,0,10.503,4.712,10.503,10.502c0,0.744-0.081,1.471-0.229,2.173l4.713,1.817L86.95,41.52L79.312,28.328z"/>

-</g>

-</svg>

+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 15.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="100px"
+	 height="100px" viewBox="0 -25.635 100 100" enable-background="new 0 -25.635 100 100" xml:space="preserve">
+<g id="docs">
+	<path fill="#C2A385" d="M86.108-9.909l5.229,9.952c0,0-1.832,0.083-5.297,1.95c-2.312,1.249-6.468,6.246-6.468,6.246L71.827-2.909
+		c0,0,4.201-3.996,6.513-5.242C81.805-10.022,86.108-9.909,86.108-9.909"/>
+	<path fill="#C2A385" d="M65.604,20.731l-3.152-9.868c0,0-5.441,3.56-8.017,4.074c-1.008,0.202-1.93,0.335-2.749,0.425
+		L65.604,20.731z"/>
+	<path fill="#C2A385" d="M72.326,23.321c0.268-0.226,0.537-0.44,0.804-0.616c3.104-2.054,6.139-3.685,6.268-3.755l1.882-1.005
+		l1.369,1.634l2.864,3.417l3.198-4.334L76.68,9.783l-8.74,11.847L72.326,23.321z"/>
+	<path fill="#C2A385" d="M39.918,10.823l4.825,1.86l3.33,0.212c0.04,0.001,0.269,0.015,0.652,0.015c0.91,0,2.798-0.072,5.196-0.551
+		c1.427-0.284,5.007-2.332,7.092-3.695l2.889-1.888l1.05,3.285l2.496,7.812l5.889-7.985l-4.625,0.163l1.348-6.225L55.133,0.593
+		l-2.095,9.667c-0.531-2.599-1.841-5.727-1.841-5.727L37.709,6.055c0,0,0.885,2.206,1.586,4.529L39.918,10.823z"/>
+	<path fill="#C2A385" d="M91.233,45.562c-1.102-0.691-2.323-1.142-3.415-1.433l-3.779,9.804c1.932,1.246,5.197,5.738,5.197,5.738
+		l7.336-9.206C96.572,50.466,93.162,46.771,91.233,45.562z"/>
+	<path fill="#C2A385" d="M93.192,32.166l-3.656,1.224c-0.019,0.007-1.779,0.613-4.117,2.069l2.817,4.868l0.626,1.08
+		c3.306-0.562,7.727-1.922,7.727-1.922l-2.332-15.261c0,0-2.934,1.277-5.852,2.221l2.318,2.765L93.192,32.166z"/>
+	<path fill="#C2A385" d="M79.272,25.999l0.864,0.334l0.46,0.801l3.503,6.05c2.646-1.636,4.611-2.287,4.611-2.287l-8.075-9.632
+		c0,0-2.584,1.391-5.376,3.188L79.272,25.999z"/>
+</g>
+<g id="trunk">
+	<circle fill="#C00000" cx="66.019" cy="52.945" r="7.877"/>
+	<circle fill="#C00000" cx="22.693" cy="52.945" r="7.877"/>
+	<path fill="#C00000" d="M22.693,42.441c1.915,0,3.705,0.522,5.251,1.421V28.001H12.191L3,37.192v14.439h9.281
+		C12.931,46.459,17.347,42.441,22.693,42.441z"/>
+	<path fill="#C00000" d="M55.925,50.075l-9.583-3.695H30.88c1.186,1.476,1.978,3.28,2.225,5.252h22.502
+		C55.674,51.1,55.78,50.58,55.925,50.075z"/>
+	<path fill="#C00000" d="M79.312,28.328L39.961,13.149l-9.384,24.335l26.381,10.174c1.824-3.115,5.198-5.218,9.062-5.218
+		c5.791,0,10.503,4.712,10.503,10.502c0,0.744-0.081,1.471-0.229,2.173l4.713,1.817L86.95,41.52L79.312,28.328z"/>
+</g>
+</svg>
 

--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -8,21 +8,13 @@
 $obj->_id = "_design/" . urlencode("app");
 $obj->language = "javascript";
 $obj->views->all->map = "function(doc) {   emit(doc._id, doc); };";
-$obj->views->byDate->map = "function(doc) {  if (doc.title != "Disclosure Log Updated") { emit(doc.date, doc); } };";
+$obj->views->byDate->map = "function(doc) {  if (doc.title != \"Disclosure Log Updated\") { emit(doc.date, doc); } };";
 $obj->views->byDateMonthYear->map = "function(doc) {   emit(doc.date, doc); };";
 $obj->views->byDateMonthYear->reduce = "_count";
 $obj->views->byAgencyID->map = "function(doc) {   emit(doc.agencyID, doc); };";
 $obj->views->byAgencyID->reduce = "_count";
-$obj->views->fieldNames->map = '
-function(doc) {
-for(var propName in doc) {
-     	emit(propName, doc._id);
-	}
-
-}';
-$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
-    return values.length;
-}';
+$obj->views->fieldNames->map = 'function(doc) { for(var propName in doc) {      	emit(propName, doc._id); 	}}';
+$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {      return values.length; }';
 // allow safe updates (even if slightly slower due to extra: rev-detection check).
 $foidb->save($obj, true);
 

--- a/documents/robots.txt
+++ b/documents/robots.txt
@@ -3,4 +3,5 @@
 
 User-agent: *
 Disallow: /admin/
+Disallow: /viewDocument.php
 Sitemap: http://disclosurelo.gs/sitemap.xml.php

--- a/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py
+++ b/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py
@@ -7,12 +7,12 @@
 #http://www.doughellmann.com/PyMOTW/abc/
 class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper):
         def getTable(self,soup):
-                return soup.find(id = "ctl00_PlaceHolderMain_PublishingPageContent__ControlWrapper_RichHtmlField").table       
+                return soup.find(id = "block-system-main").table       
         def getColumnCount(self):
-                return 7
+                return 2
         def getColumns(self,columns):
-                (id, date, title, description,link,deldate, notes) = columns
-                return (id, date, title, description, notes)
+                (date, title) = columns
+                return (date, date, title, title, None)
 
 if __name__ == '__main__':
     print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper)

--- a/documents/sitemap.xml.php
+++ b/documents/sitemap.xml.php
@@ -1,30 +1,48 @@
 <?php
 
 include ('../include/common.inc.php');
-$last_updated = date('Y-m-d', @filemtime('cbrfeed.zip'));
 header("Content-Type: text/xml");
 echo "<?xml version='1.0' encoding='UTF-8'?>";
 echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
 echo " <url><loc>" . local_url() . "index.php</loc><priority>1.0</priority></url>\n";
 foreach (scandir("./") as $file) {
-    if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php")
+    if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php"&& $file != "viewDocument.php")
         echo " <url><loc>" . local_url() . "$file</loc><priority>0.6</priority></url>\n";
+	}
 }
 $agenciesdb = $server->get_db('disclosr-agencies');
+$foidocsdb = $server->get_db('disclosr-foidocuments');
 try {
     $rows = $agenciesdb->get_view("app", "byCanonicalName")->rows;
     foreach ($rows as $row) {
         echo '<url><loc>' . local_url() . 'agency.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
     }
+	unset($rows);
+	$rows = null;
 } catch (SetteeRestClientException $e) {
     setteErrorHandler($e);
 }
-$foidocsdb = $server->get_db('disclosr-foidocuments');
+
+foreach (range(0, 8) as $number) {
 try {
-    $rows = $foidocsdb->get_view("app", "all")->rows;
+    $rows = $foidocsdb->get_view("app", "all", Array($number,$number+1))->rows;
     foreach ($rows as $row) {
         echo '<url><loc>' . local_url() . 'view.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
     }
+	unset($rows);
+	$rows = null;
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+}
+
+try {
+    $rows = $foidocsdb->get_view("app", "all", Array('9','fffffffff'))->rows;
+    foreach ($rows as $row) {
+        echo '<url><loc>' . local_url() . 'view.php?id=' . $row->value->_id . "</loc><priority>0.3</priority></url>\n";
+    }
+	unset($rows);
+	$rows = null;
 } catch (SetteeRestClientException $e) {
     setteErrorHandler($e);
 }

--- /dev/null
+++ b/exportAgencies.csv.php
@@ -1,1 +1,88 @@
+<?php
+// use https://github.com/okfn/publicbodies/blob/master/data/nz.csv format
+include_once("include/common.inc.php");
 
+setlocale(LC_CTYPE, 'C');
+
+$headers = Array("title","abbr","key","category","parent","parent_key","description","url","jurisdiction","jurisdiction_code","source","source_url","address","contact","email","tags","created_at","updated_at");
+
+$db = $server->get_db('disclosr-agencies');
+
+
+$foiEmail = Array();
+try {
+    $rows = $db->get_view("app", "foiEmails", null, true)->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        $foiEmail[$row->key] = $row->value;
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+    die();
+}
+
+$fp = fopen('php://output', 'w');
+if ($fp && $db) {
+    header('Content-Type: text/csv; charset=utf-8');
+    header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
+    header('Pragma: no-cache');
+    header('Expires: 0');
+    fputcsv($fp, $headers);
+    try {
+        $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+        //print_r($rows);
+        foreach ($agencies as $agency) {
+            // print_r($agency);
+
+            if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
+                $row = Array();
+                $row["title"] = trim($agency->value->name);
+                $row["abbr"] = (isset($agency->value->shortName) ? $agency->value->shortName : "");
+                $row["key"] = (isset($agency->value->shortName) ? "au/".strtolower($agency->value->shortName) : "");
+                $row["category"] ="";
+                 $row["parent"] ="";
+                 $row["parentkey"] ="";
+                $row["description"] = (isset($agency->value->description) ? $agency->value->description : "");
+                $row["url"] = (isset($agency->value->website) ? $agency->value->website : "");
+                $row["jurisdiction"] = "Australia";
+                $row["jurisdiction_code"] = "au";
+
+                 $row["source"] ="";
+                 $row["source_url"] ="";
+                 $row["address"] ="";
+                 $row["contact"] ="";
+
+                $row["email"] = (isset($agency->value->foiEmail) ? $agency->value->foiEmail : "");
+                 $row["tags"] ="";
+                 $row["created_at"] ="";
+                 $row["updated_at"] ="";
+
+
+                $otherBodies = Array();
+                if (isset($agency->value->foiBodies)) {
+                    $otherBodies = array_merge($otherBodies, $agency->value->foiBodies);
+                }
+                if (isset($agency->value->positions)) {
+                    $positions = Array();
+                    foreach ($agency->value->positions as $position) {
+                        $positions[] = "Office of the ".$position;
+                    }
+                    $otherBodies = array_merge($otherBodies, $positions);
+                }
+                sort($otherBodies);
+                if (count($otherBodies) > 0) {
+                    $row["description"] .= "<br/> This department also responds to requests for information held by " . implode(", ", $otherBodies);
+                }
+
+
+                fputcsv($fp, array_values($row));
+            }
+        }
+    } catch (SetteeRestClientException $e) {
+        setteErrorHandler($e);
+    }
+
+    die;
+}
+?>
+

--- a/getAgency.php
+++ b/getAgency.php
@@ -188,7 +188,7 @@
           </div><!--/.well -->
         </div><!--/span-->
         <div class="span9">';
-    echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization" about="#' . $row['_id'] . '">';
+    echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization org:Organization" about="#' . $row['_id'] . '">';
     echo '<div class="hero-unit">
             <h1 itemprop="name">' . $row['name'] . '</h1>';
     if (isset($row['description'])) {

--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -10,6 +10,7 @@
         || strstr($_SERVER['PHP_SELF'], "documents/")
 	|| $_SERVER['SERVER_NAME'] == "disclosurelo.gs"
 	|| $_SERVER['SERVER_NAME'] == "www.disclosurelo.gs"
+	|| $_SERVER['SERVER_NAME'] == "direct.disclosurelo.gs"
         )
     $basePath = "../";
 

--- a/include/template.inc.php
+++ b/include/template.inc.php
@@ -44,7 +44,7 @@
             <![endif]-->
 
         </head>
-        <body xmlns:schema="http://schema.org/"  xmlns:foaf="http://xmlns.com/foaf/0.1/">
+        <body xmlns:schema="http://schema.org/"  xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:org="http://www.w3.org/ns/org#" xmlns:skos="http://www.w3.org/2004/02/skos/core#">
 <div class="navbar navbar-inverse navbar-fixed-top">
                 <div class="navbar-inner">
                     <div class="container-fluid">

file:a/robots.txt -> file:b/robots.txt
--- a/robots.txt
+++ b/robots.txt
@@ -3,4 +3,5 @@
 
 User-agent: *
 Disallow: /admin/
+Disallow: /viewDocument.php
 Sitemap: http://orgs.disclosurelo.gs/sitemap.xml.php

--- a/schemas/agency.json.php
+++ b/schemas/agency.json.php
@@ -4,7 +4,7 @@
     "description" => "Representation of government agency and online transparency measures",
     "type" => "object",
     "properties" => Array(
-        "name" => Array("type" => "string", "required" => true, "x-itemprop" => "name", "x-property" => "schema:name foaf:name", "x-title" => "Name", "description" => "Name, most recent and broadest"),
+        "name" => Array("type" => "string", "required" => true, "x-itemprop" => "name", "x-property" => "schema:name foaf:name skos:prefLabel ", "x-title" => "Name", "description" => "Name, most recent and broadest"),
         "shortName" => Array("type" => "string", "required" => false, "x-title" => "Short Name", "description" => "Name shortened, usually to an acronym"),
    "description" => Array("type" => "string", "required" => false, "x-title" => "Description", "description" => "Description of roles and responsiblities of organisation"),
         "foiEmail" => Array("type" => "string", "required" => false, "x-title" => "FOI Contact Email", "x-itemprop" => "email", "description" => "FOI contact email if not foi@"),
@@ -14,15 +14,15 @@
             "items" => Array("type" => "string")),
                 "positions" => Array("type" => "array", "required" => true, "x-title" => "Political Positions", "description" => "Ministers and Parliamentary Secretaries",
             "items" => Array("type" => "string")),
-        "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies","x-property"=>"schema:members foaf:knows",  "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous",
+        "foiBodies" => Array("type" => "array", "required" => true, "x-title" => "FOI Bodies","x-property"=>"schema:members foaf:knows org:hasSubOrganization",  "description" => "Organisational units within this agency that are subject to FOI Act but are not autonomous",
             "items" => Array("type" => "string")),
         "legislation" => Array("type" => "array", "required" => true, "x-title" => "Legislation", "description" => "Legislation administered by or created for the establishment of this organisation",
             "items" => Array("type" => "string")),
-        "orgType" => Array("type" => "string", "required" => true, "x-title" => "Organisation Type", "description" => "Org type based on legal formation via FMA/CAC legislation etc."),
-        "parentOrg" => Array("type" => "string", "required" => true, "x-title" => "Parent Organisation", "description" => "Parent organisation, usually a department of state"),
+        "orgType" => Array("type" => "string", "required" => true, "x-title" => "Organisation Type", "x-property" => "org:classification", "description" => "Org type based on legal formation via FMA/CAC legislation etc."),
+        "parentOrg" => Array("type" => "string", "required" => true, "x-title" => "Parent Organisation", "x-property" => "org:subOrganizationOf", "description" => "Parent organisation, usually a department of state"),
         "website" => Array("type" => "string", "required" => true, "x-title" => "Website", "x-itemprop" => "url", "x-property" => "schema:url foaf:homepage", "description" => "Website URL"),
-        "abn" => Array("type" => "string", "required" => true, "x-title" => "Australian Business Number", "description" => "ABN from business register"),
-        "established" => Array("type" => "string", "required" => true, "x-title" => "Date established", "description" => "Date established"),
+        "abn" => Array("type" => "string", "required" => true, "x-title" => "Australian Business Number", "x-property" => "org:identifier", "description" => "ABN from business register"),
+        "established" => Array("type" => "string", "required" => true, "x-title" => "Date established", "x-property" => "schema:foundingDate", "description" => "Date established"),
         "employees" => Array("type" => "string", "required" => true, "x-title" => "2010-2011 employees", "description" => "2010-2011 employees"),
         "contractListURL" => Array("type" => "string", "required" => true,  "x-title" => "Contract Listing", "description" => "Departmental and agency contracts, <a href='http://www.aph.gov.au/senate/pubs/standing_orders/d05.htm'>mandated by the Senate</a>" ),
         "budgetURL" => Array("type" => "string", "required" => true,"x-title" => "Budget", "description" => "Portfolio Budget Statements and Portfolio Additional Estimates Statements"),

--- a/sitemap.xml.php
+++ b/sitemap.xml.php
@@ -7,7 +7,7 @@
 echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
 echo " <url><loc>" . local_url() . "index.php</loc><priority>1.0</priority></url>\n";
 foreach (scandir("./") as $file) {
-    if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php")
+    if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php"&& $file != "viewDocument.php")
         echo " <url><loc>" . local_url() . "$file</loc><priority>0.3</priority></url>\n";
 }