Scrape required and chart of complied features views
Scrape required and chart of complied features views


Former-commit-id: 1f70b43713b7686e9f7a8a0f6a5aced655d53221

--- a/.gitmodules
+++ b/.gitmodules
@@ -4,4 +4,10 @@
 [submodule "couchdb/settee"]
 	path = couchdb/settee
 	url = https://github.com/inadarei/settee.git
+[submodule "lib/springy"]
+	path = lib/springy
+	url = https://github.com/dhotson/springy.git
+[submodule "lib/php-diff"]
+	path = lib/php-diff
+	url = https://github.com/chrisboulton/php-diff.git
 

file:b/admin/cacfma.csv (new)
--- /dev/null
+++ b/admin/cacfma.csv
@@ -1,1 +1,191 @@
+AAF Company,82?008?629?490
+Aboriginal Hostels Limited ,47?008?504?587
+Administrative Appeals Tribunal,90?680?970?626
+Aged Care Standards and Accreditation Agency Ltd,64?079?618?652
+Airservices Australia  ,59?698?720?886
+Albury-Wodonga Development Corporation  ,71?893?478?442
+Anindilyakwa Land Council ,45?175?406?445
+Army and Air Force Canteen Service ,69?289?134?420
+ASC Pty Ltd ,64?008?605?034
+Attorney-General's Department,92?661?124?436
+Australia Business Arts Foundation Ltd ,88?072?479?835
+Australia Council,38?392?626?187
+Australian Agency for International Development (AusAID),62?921?558?838
+Australian Broadcasting Corporation,52?429?278?345
+Australian Bureau of Statistics,26?331?428?522
+Australian Centre for International Agricultural Research (ACIAR),34?864?955?427
+Australian Commission for Law Enforcement Integrity (ACLEI),78?796?734?093
+Australian Commission on Safety and Quality in Health Care,97250687371
+Australian Communications and Media Authority (ACMA),55?386?169?386
+Australian Competition and Consumer Commission,94?410?483?623
+Australian Crime Commission,11?259?448?410
+"Australian Curriculum, Assessment and Reporting Authority ",54?735?928?084
+Australian Customs and Border Protection Service,66?015?286?036
+Australian Electoral Commission,21?133?285?851
+Australian Federal Police,17?864?931?143
+"Australian Film, Television and Radio School",19?892?732?021
+Australian Fisheries Management Authority,81?098?497?517
+Australian Government Solicitor,69?405?937?639
+Australian Hearing Services ,80?308?797?003
+Australian Human Rights Commission,47?996?232?602
+Australian Industry Development,55?085?059?559
+Australian Institute for Teaching and School Leadership Limited,17?117?362?740
+Australian Institute of Aboriginal and Torres Strait Islander Studies,62?020?533?641
+Australian Institute of Criminology,63257175248
+Australian Institute of Family Studies (AIFS),64?001?053?079
+Australian Institute of Health and Welfare  ,16?515?245?497
+Australian Institute of Marine Science,78?961?616?230
+Australian Law Reform Commission,88913413914
+Australian Learning and Teaching Council Limited ,30?109?826?628
+Australian Maritime Safety Authority,65?377?938?320
+Australian Military Forces Relief Trust Fund ,52?168?913?646
+Australian National Audit Office ,33?020?645?631
+Australian National Maritime Museum,35?023?590?988
+Australian National Preventive Health Agency (ANPHA),33?965?140?953
+Australian National University,52?234?063?906
+Australian Nuclear Science and Technology Organisation ,47?956?969?590
+Australian Office of Financial Management (AOFM),13?059?525?039
+Australian Pesticides and Veterinary Medicines Authority (APVMA),19?495?043?447
+Australian Postal Corporation,28?864?970?579
+Australian Prudential Regulation Authority (APRA),79?635?582?658
+Australian Public Service Commission (APS Commission),99?470?863?260
+Australian Radiation Protection and Nuclear Safety Agency (ARPANSA),61?321?195?155
+Australian Rail Track Corporation Limited ,75?081?455?754
+Australian Reinsurance Pool Corporation,74?807?136?872
+Australian Research Council,35?201?451?156
+Australian River Co. Limited,94?008?654?206
+Australian Secret Intelligence Service,49?667?785?014
+Australian Securities and Investments Commission,86?768?265?615
+Australian Security Intelligence Organisation,37?467?566?201
+Australian Skills Quality Authority (National Vocational Education and Training Regulator),72581678650
+Australian Solar Institute Limited ,65138300688
+Australian Sports Anti-Doping Authority (ASADA),91?592?527?503
+Australian Sports Commission,67374695240
+Australian Sports Foundation Limited ,27?008?613?858
+Australian Strategic Policy Institute Limited ,77?097?369?045
+Australian Taxation Office,51?824?753?556
+Australian Trade Commission (Austrade),11?764?698?227
+Australian Transaction Reports and Analysis Centre (AUSTRAC),32?770?513?371
+Australian Transport Safety Bureau (ATSB),86?267?354?017
+Australian War Memorial ,64?909?221?257
+Bundanon Trust,72?058?829?217
+Bureau of Meteorology,92?637?533?532
+Cancer Australia,21?075?951?918
+Central Land Council,71?979?619?393
+Civil Aviation Safety Authority,44?808?014?470
+Coal Mining Industry (Long Service Leave Funding) Corporation,12?039?670?644
+Comcare  ,41?640?788?304
+Commonwealth Grants Commission,64?703?642?210
+Commonwealth Scientific and Industrial Research Organisation,41?687?119?230
+Commonwealth Superannuation Corporation ,48882817243
+ComSuper,77?310?752?950
+Corporations and Markets Advisory Committee (CAMAC),41?574?479?010
+Cotton Research and Development Corporation,71?054?238?316
+CrimTrac Agency,17?193?904?699
+Defence Housing Australia,72?968?504?934
+"Department of Agriculture, Fisheries and Forestry ",24?113?085?695
+"Department of Broadband, Communications and the Digital Economy",51?491?646?726
+Department of Climate Change and Energy Efficiency,50?182?626?845
+"Department of Education, Employment and Workplace Relations",63?578?775?294
+"Department of Families, Housing, Community Services and Indigenous Affairs",36?342?015?855
+Department of Finance and Deregulation,61?970?632?495
+Department of Foreign Affairs and Trade,47?065?634?525
+Department of Health and Ageing,83?605?426?759
+Department of Human Services,90?794?605?008
+Department of Immigration and Citizenship,33?380?054?835
+Department of Infrastructure and Transport,86?267?354?017
+"Department of Innovation, Industry, Science and Research",74?599?608?295
+Department of Parliamentary Services,52?997?141?147
+"Department of Regional Australia, Regional Development and Local Government",37?862?725?624
+"Department of Resources, Energy and Tourism",46?252?861?927
+"Department of Sustainability, Environment, Water, Population and Communities",34?190?894?983
+Department of the House of Representatives,18?526?287?740
+Department of the Prime Minister and Cabinet,18?108?001?191
+Department of the Senate,23?991?641?527
+Department of the Treasury,92?802?414?793
+Department of Veterans' Affairs,23?964?290?824
+Director of National Parks ,13?051?694?963
+Equal Opportunity for Women in the Workplace Agency,47?641?643?874
+Export Finance and Insurance Corporation,96?874?024?697
+Fair Work Australia (FWA),93?614?579?199
+Family Court of Australia,63?684?208?971
+Federal Court of Australia,49?110?847?399
+Federal Magistrates Court of Australia,60?265?617?271
+Fisheries Research and Development Corporation,74?311?094?913
+Food Standards Australia New Zealand,20?537?066?246
+Future Fund Management Agency,53?156?699?293
+General Practice Education and Training Limited,95?095?433?140
+Geoscience Australia,80?091?799?039
+Grains Research and Development Corporation ,55?611?223?291
+Grape and Wine Research and Development Corporation,72?618?007?571
+Great Barrier Reef Marine Park Authority,12?949?356?885
+Health Workforce Australia,21?295?050?589
+HIH Claims Support Limited,92?096?857?635
+IIF Investments Pty Limited,55?082?153?884
+Indigenous Business Australia,25?192?932?833
+Indigenous Land Corporation,59?912?679?254
+Insolvency and Trustee Service Australia (ITSA),63?384?330?717
+Inspector-General of Taxation,51?248?702?319
+Interim Independent Hospital Pricing Authority,27598959960
+IP Australia,38?113?072?755
+Low Carbon Australia Limited,63?097?727?968
+Medibank Private Limited ,47?080?890?259
+Migration Review Tribunal and Refugee Review Tribunal ,50?760?799?564
+Murray-Darling Basin Authority,13?679?821?382
+National Archives of Australia,36?889?228?992
+National Australia Day Council Limited ,76?050?300?626
+National Blood Authority,87?361?602?478
+National Breast and Ovarian Cancer Centre,85?094?118?902
+National Capital Authority,75?149?374?427
+National Competition Council ,56?552?760?098
+National Film and Sound Archive,41?251?017?588
+National Gallery of Australia,27?855?975?449
+National Health and Medical Research Council (NHMRC),88?601?010?284
+National Library of Australia ,28?346?858?075
+National Museum of Australia ,70?592?297?967
+National Native Title Tribunal,70?238?042?351
+National Offshore Petroleum Safety Authority (NOPSA),22?385?178?289
+National Water Commission ,94?364?176?431
+NBN Co Limited,86?136?533?741
+Northern Land Council,56?327?515?336
+Office of National Assessments,87?904?367?991
+Office of Parliamentary Counsel,41?425?630?817
+Office of the Auditing and Assurance Standards Board ,80?959?780?601
+Office of the Australian Accounting Standards Board (AASB),92?702?019?575
+Office of the Australian Building and Construction Commissioner,68?003?725?098
+Office of the Australian Information Commissioner ,85249230937
+Office of the Commonwealth Ombudsman,53?003?678?148
+Office of the Director of Public Prosecutions,41?036?606?436
+Office of the Fair Work Ombudsman,71?141?751?477
+Office of the Inspector-General of Intelligence and Security,67?332?668?643
+Office of the Official Secretary to the Governor-General,67?582?329?284
+Office of the Renewable Energy Regulator,68?574?011?917
+Old Parliament House,30?620?774?963
+Organ and Tissue Authority (Australian Organ and Tissue Donation and Transplantation Authority),56?253?405?315
+Outback Stores Pty Ltd ,63120661234
+Private Health Insurance Administration Council ,50?831?782?014
+Private Health Insurance Ombudsman,61?673?137?709
+Productivity Commission,78?094?372?050
+Professional Services Review Scheme,45?307?308?260
+RAAF Welfare Recreational Company ,45?008?499?303
+Reserve Bank of Australia,50?008?559?486
+Royal Australian Air Force Veterans' Residences Trust Fund ,40?594?141?285
+Royal Australian Air Force Welfare Trust Fund ,24?616?803?717
+Royal Australian Mint,45?852?104?259
+Royal Australian Navy Central Canteens Board,50?616?294?781
+Royal Australian Navy Relief Trust Fund ,49?934?525?476
+Rural Industries Research and Development Corporation,25?203?754?319
+Safe Work Australia,81?840?374?163
+Screen Australia ,46?741?353?180
+"Seafarers Safety, Rehabilitation and Compensation Authority (Seacare Authority)",32?745?854?352
+Special Broadcasting Service Corporation,91?314?398?574
+Sugar Research and Development Corporation,41?343?997?980
+Sydney Harbour Federation Trust,14?178?614?905
+Tertiary Education Quality and Standards Agency,50658250012
+Tiwi Land Council,86?106?441?085
+Torres Strait Regional Authority,57?155?285?807
+Tourism Australia ,99?657?548?712
+Wheat Exports Australia,40?485?918?341
+Wine Australia Corporation ,59?728?300?326
+Wreck Bay Aboriginal Community Council,62?564?797?956
 

file:b/admin/import.php (new)
--- /dev/null
+++ b/admin/import.php
@@ -1,1 +1,34 @@
+<?php
 
+require_once '../include/common.inc.php';
+try {
+    $server->create_db('disclosr-agencies');
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+$db = $server->get_db('disclosr-agencies');
+createAgencyDesignDoc();
+$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");
+$namesQ = 'select agency.abn, string_agg("agencyName",\'|\') as names from agency inner join agency_nametoabn on agency.abn::text = agency_nametoabn.abn group by agency.abn;';
+$abntonames = Array();
+foreach ($conn->query($namesQ) as $row) {
+    $abntonames[$row['abn']] = explode("|", $row['names']);
+}
+$result = $conn->query("select * from agency");
+while ($agency = $result->fetch(PDO::FETCH_ASSOC)) {
+    $agency['_id'] = md5($agency['abn']);
+    $agency['otherNames'] = $abntonames[$agency['abn']];
+    if (sizeof($abntonames[$agency['abn']]) == 1)
+        $agency['name'] = $abntonames[$agency['abn']][0];
+    $agency["lastScraped"] = "1/1/1970";
+    $agency["scrapeDepth"] = 1;
+    try {
+        $doc = $db->save($agency);
+        //print_r($doc);
+        echo $agency['abn'] . " imported \n<br>";
+    } catch (SetteeRestClientException $e) {
+        setteErrorHandler($e);
+    }
+}
+?>
+

--- /dev/null
+++ b/admin/refreshDesignDoc.php
@@ -1,1 +1,7 @@
+<?php
 
+require_once '../include/common.inc.php';
+$db = $server->get_db('disclosr-agencies');
+createAgencyDesignDoc();
+?>
+

--- /dev/null
+++ b/admin/resolveConflicts.php
@@ -1,1 +1,43 @@
+<?php
 
+include_once('../include/common.inc.php');
+include_header();
+// Include the diff class
+echo '<STYLE TYPE="text/css">
+<!--
+  @import url(../lib/php-diff/example/styles.css);
+-->
+</STYLE>
+    ';
+require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff.php';
+// Generate a side by side diff
+require_once dirname(__FILE__) . '/../lib/php-diff/lib/Diff/Renderer/Html/SideBySide.php';
+$renderer = new Diff_Renderer_Html_SideBySide;
+
+
+
+$db = $server->get_db('disclosr-agencies');
+$docs = Array();
+try {
+    $rows = $db->get_view("app", "getConflicts")->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        echo '<h2>' . $row->id . '</h2>';
+        echo "Comparing " . $row->value[0] . " and " . $row->value[1];
+        $docA = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[0])));
+        $docB = explode(",", json_encode($db->get($row->id . "?rev=" . $row->value[1])));
+        // Options for generating the diff
+        $options = array(
+                //'ignoreWhitespace' => true,
+                //'ignoreCase' => true,
+        );
+
+        // Initialize the diff class
+        $diff = new Diff($docA, $docB, $options);
+        echo $diff->Render($renderer);
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+include_footer();
+?>

file:b/admin/verify.php (new)
--- /dev/null
+++ b/admin/verify.php
@@ -1,1 +1,56 @@
+<?php
 
+include_once('../include/common.inc.php');
+include_header();
+
+$db = $server->get_db('disclosr-agencies');
+$docs = Array();
+try {
+    $rows = $db->get_view("app", "byABN")->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        $docs["a" . $row->key] = $row->value;
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+//print_r($docs);
+$row = 1;
+if (($handle = fopen("cacfma.csv", "r")) !== FALSE) {
+    while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
+        $row++;
+        echo $data[0] . "   " . str_replace("?", "", $data[1]) . "<br />\n";
+        $name = $data[0];
+        $abn = trim(str_replace("?", "", $data[1]));
+        $aabn = "a".$abn;
+        if (isset($docs[$aabn])) {
+            echo "Existing agency ABN detected<br>";
+            if (!in_array($name, object_to_array($docs[$aabn]->otherNames)) && $name != $docs[$aabn]->name) {
+                $docs[$aabn]->otherNames[] = $name;
+                try {
+                    $docs[$aabn] = $db->save($docs[$aabn]);
+                    //print_r($doc);
+                    echo $abn . " additional names imported \n<br>";
+                } catch (SetteeRestClientException $e) {
+                    setteErrorHandler($e);
+                }
+            }
+        } else {
+            echo "New agency ABN detected<br>";
+            $agency['_id'] = md5($aabn);
+            $agency['name'] = $name;
+            $agency["abn"] = $abn;
+            try {
+                $doc = $db->save($agency);
+                print_r($doc);
+                echo $abn . " imported \n<br>";
+            } catch (SetteeRestClientException $e) {
+                setteErrorHandler($e);
+            }
+        }
+        echo "<hr>";
+    }
+    fclose($handle);
+}
+include_footer();
+?>

--- /dev/null
+++ b/alaveteli/exportAgencies.csv.php
@@ -1,1 +1,107 @@
+<?php
 
+include_once("../include/common.inc.php");
+
+function shortName($name) {
+    $name = trim($name);
+    if (strstr($name,"Minister ") || strstr($name,"Treasurer") || strstr($name,"Parliamentary Secretary")) {
+        $badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
+        return str_replace($badWords,"",$name);
+    }
+            
+    else {
+    $out = Array();
+    preg_match_all('/[A-Z]/', $name, $out);
+    return implode("", $out[0]);
+    }
+}
+
+setlocale(LC_CTYPE, 'C');
+
+$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");
+
+$db = $server->get_db('disclosr-agencies');
+
+$tag = Array();
+try {
+    $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        $tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+    die();
+}
+
+$foiEmail = Array();
+try {
+    $rows = $db->get_view("app", "foiEmails", null, true)->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        $foiEmail[$row->key] = $row->value;
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+    die();
+}
+
+$fp = fopen('php://output', 'w');
+if ($fp && $db) {
+    header('Content-Type: text/csv; charset=utf-8');
+    header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
+    header('Pragma: no-cache');
+    header('Expires: 0');
+    fputcsv($fp, $headers);
+    try {
+        $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+        //print_r($rows);
+        foreach ($agencies as $agency) {
+            // print_r($agency);
+
+            if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
+                $row = Array();
+                $row["#id"] = $agency->id;
+                $row["name"] = trim($agency->value->name);
+                if (isset($agency->value->foiEmail)) {
+                    $row["request_email"] = $agency->value->foiEmail;
+                } else {
+                    if ($agency->value->orgType == "FMA-DepartmentOfState") {
+                        $row["request_email"] = "foi@" . GetDomain($agency->value->website);
+                    } else {
+                        $row["request_email"] = $foiEmail[$agency->value->parentOrg];
+                    }
+                }
+                if (isset($agency->value->shortName)) {
+                    $row["short_name"] = $agency->value->shortName;
+                } else {
+                    $row["short_name"] = shortName($agency->value->name);
+                }
+                $row["notes"] = "";
+                $row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
+                $row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
+                if ($agency->value->orgType == "FMA-DepartmentOfState") {
+                    $row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
+                } else {
+                    $row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
+                }
+
+                fputcsv($fp, array_values($row));
+
+                if (isset($agency->value->foiBodies)) {
+                    foreach ($agency->value->foiBodies as $foiBody) {
+                        $row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
+                        $row["short_name"] = shortName($foiBody);
+                        fputcsv($fp, array_values($row));
+                    }
+                }
+            }
+        }
+    } catch (SetteeRestClientException $e) {
+        setteErrorHandler($e);
+    }
+
+    die;
+}
+?>
+

--- /dev/null
+++ b/alaveteli/exportCategories.rb.php
@@ -1,1 +1,24 @@
+<?php
 
+include_once("../include/common.inc.php");
+setlocale(LC_CTYPE, 'C');
+header('Content-Type: text/csv');
+header('Content-Disposition: attachment; filename="public_body_categories_en.rb"');
+header('Pragma: no-cache');
+header('Expires: 0');
+echo 'PublicBodyCategories.add(:en, [' . PHP_EOL;
+echo '    "Portfolios",' . PHP_EOL;
+$db = $server->get_db('disclosr-agencies');
+
+try {
+    $rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
+    //print_r($rows);
+    foreach ($rows as $row) {
+        echo '        [ "' . phrase_to_tag(dept_to_portfolio($row->key)) . '","' . dept_to_portfolio($row->key) . '","part of the ' . dept_to_portfolio($row->key) . ' portfolio" ],' . PHP_EOL;
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+echo '])';
+?>
+

--- a/getAgency.php
+++ b/getAgency.php
@@ -4,16 +4,36 @@
 include_header();
 
 function displayValue($key, $value, $mode) {
+    global $db, $schemas;
     if ($mode == "view") {
+
+        echo "<tr>";
+
+        echo "<td>" . $schemas['agency']["properties"][$key]['x-title'] . "<br><small>" . $schemas['agency']["properties"][$key]['description'] . "</small></td><td>";
         if (is_array($value)) {
-            echo "<tr><td>$key</td><td><ol>";
+            echo "<ol>";
             foreach ($value as $subkey => $subvalue) {
-                echo "<li>$subvalue</li>";
+                if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
+                echo '<li itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
+            } else {
+                echo "<li>";
+            }
+                echo "$subvalue</li>";
             }
             echo "</ol></td></tr>";
         } else {
-            echo "<tr><td>$key</td><td>$value</td></tr>";
+            if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
+                echo '<span itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '">';
+            } else {
+                echo "<span>";
+            }
+              if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
+                    echo "<a href='$value'>view</a></span>";
+                } else {
+            echo "$value</span>";
+                }
         }
+        echo "</td></tr>";
     }
     if ($mode == "edit") {
         if (is_array($value)) {
@@ -30,15 +50,23 @@
         } else {
             if (strpos($key, "_") === 0) {
                 echo"<input type='hidden' id='$key' name='$key' value='$value'/>";
-            } if (strpos($key, "has") === 0) {
-                echo "<label for='$key'><input type='checkbox' id='$key' name='$key' checked='$value'> $key</label>";
+            } else if ($key == "parentOrg") {
+                echo "<label for='$key'>$key</label><select  id='$key' name='$key'><option value=''> Select... </option>";
+                $rows = $db->get_view("app", "byDeptStateName")->rows;
+                //print_r($rows);
+                foreach ($rows as $row) {
+                    echo "<option value='{$row->value}'" . (($row->value == $value) ? "SELECTED" : "") . " >" . str_replace("Department of ", "", $row->key) . "</option>";
+                }
+                echo" </select>";
+            } else if (strpos($key, "has") === 0) {
+                echo "<label for='$key'><input type='checkbox' id='$key' name='$key' " . (($value == 'on' || $value == 'true') ? "checked='$value'" : "") . "> $key</label>";
             } else {
                 echo "<label>$key</label><input  class='input-text' type='text' id='$key' name='$key' value='$value'/>";
-                if ((strpos($key,"URL") > 0 || $key == 'website')&& $value != "") {
-                   echo "<a href='$value'>view</a>"; 
+                if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
+                    echo "<a href='$value'>view</a>";
                 }
                 if ($key == 'abn') {
-                   echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>";
+                    echo "<a href='http://www.abr.business.gov.au/SearchByAbn.aspx?SearchText=$value'>view abn</a>";
                 }
             }
         }
@@ -47,10 +75,22 @@
 }
 
 function addDefaultFields($row) {
-    $defaultFields = Array("name");
+    global $schemas;
+    $defaultFields = array_keys($schemas['agency']['properties']);
     foreach ($defaultFields as $defaultField) {
-        if (!isset($row[$defaultField]))
-            $row[$defaultField] = "";
+        if (!isset($row[$defaultField])) {
+            if ($schemas['agency']['properties'][$defaultField]['type'] == "string") {
+                if (strpos($defaultField, "has") === 0) {
+                    $row[$defaultField] = "false";
+                } else {
+                    $row[$defaultField] = "";
+                }
+            }
+            if ($schemas['agency']['properties'][$defaultField]['type'] == "array") {
+
+                $row[$defaultField] = Array("");
+            }
+        }
     }
     return $row;
 }
@@ -66,20 +106,33 @@
     //print_r($row);
     if (sizeof($_POST) > 0) {
         //print_r($_POST);
+        foreach ($_POST as $postkey => $postvalue) {
+            if ($postvalue == "") {
+                unset($_POST[$postkey]);
+            }
+            if (is_array($postvalue) && count($postvalue) == 1 && $postvalue[0] == "") {
+                unset($_POST[$postkey]);
+            }
+        }
         if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) {
             echo "Edited version was latest version, continue saving";
             $newdoc = $_POST;
             $newdoc['metadata']['lastModified'] = time();
             $row = $db->save($newdoc);
         } else {
-            echo "ALERT doc revised by someone else while editing.";
+            echo "ALERT doc revised by someone else while editing. Document not saved.";
         }
     }
 
-    $mode = "edit";
-    $row = addDefaultFields(object_to_array($row));
+    $mode = "view";
+    if ($mode == "edit") {
+        $row = addDefaultFields(object_to_array($row));
+    } else {
+         $row = object_to_array($row);
+    }
+    
     if ($mode == "view") {
-        echo '<table width="100%">';
+        echo '<div itemscope itemtype ="http://schema.org/GovernmentOrganisation"><table width="100%">';
         echo '<tr> <td colspan="2"><h3>' . $row['name'] . "</h3></td></tr>";
         echo "<tr><th>Field Name</th><th>Field Value</th></tr>";
     }
@@ -104,32 +157,40 @@
             };
         </script>
         <form id="editform" class="nice" method="post">
-            <?php
+        <?php
 
+    }
+    foreach ($row as $key => $value) {
+        echo displayValue($key, $value, $mode);
+    }
+    if ($mode == "view") {
+        echo "</table></div>";
+    }
+    if ($mode == "edit") {
+        echo '<input id="submitbutton" type="submit"/></form>';
+    }
+} else {
+
+    try {
+        /* $rows = $db->get_view("app", "showNamesABNs")->rows;
+          //print_r($rows);
+          foreach ($rows as $row) {
+          //   print_r($row);
+          echo '<li><a href="getAgency.php?id=' . $row->key . '">' .
+          (isset($row->value->name) && $row->value->name != "" ? $row->value->name : "NO NAME " . $row->value->abn)
+          . '</a></li>';
+          } */
+        $rows = $db->get_view("app", "byName")->rows;
+        //print_r($rows);
+        foreach ($rows as $row) {
+            //   print_r($row);
+            echo '<li itemscope itemtype="http://schema.org/GovernmentOrganization"><a href="getAgency.php?id=' . $row->value . '" itemprop="url"><span itemprop="name">' .
+            $row->key
+            . '</span></a></li>';
         }
-        foreach ($row as $key => $value) {
-            echo displayValue($key, $value, $mode);
-        }
-        if ($mode == "view") {
-            echo "</table>";
-        }
-        if ($mode == "edit") {
-            echo '<input id="submitbutton" type="submit"/></form>';
-        }
-    } else {
-
-        try {
-            $rows = $db->get_view("app", "showNamesABNs")->rows;
-            //print_r($rows);
-            foreach ($rows as $row) {
-                //   print_r($row);
-                echo '<li><a href="getAgency.php?id=' . $row->key . '">' .
-                (isset($row->value->name) && $row->value->name != "" ? $row->value->name : "NO NAME " . $row->value->abn)
-                . '</a></li>';
-            }
-        } catch (SetteeRestClientException $e) {
-            setteErrorHandler($e);
-        }
+    } catch (SetteeRestClientException $e) {
+        setteErrorHandler($e);
     }
-    include_footer();
-    ?>
+}
+include_footer();
+?>

file:b/graph.php (new)
--- /dev/null
+++ b/graph.php
@@ -1,1 +1,93 @@
+<?php
+include_once('include/common.inc.php');
+//include_header();
+$format = "html";
+if (isset($_REQUEST['format'])) {
+    $format = $_REQUEST['format'];
+}
 
+function add_node($id, $label) {
+    global $format;
+    if ($format == "html") {
+        echo "nodes[\"$id\"] = graph.newNode({label: \"$label\"});" . PHP_EOL;
+    }
+     if ($format == "dot" && $label != "") {
+         echo "$id [label=\"$label\"];". PHP_EOL;
+     }
+}
+
+function add_edge($from, $to, $color) {
+    global $format;
+    if ($format == "html"