better date parser
better date parser


Former-commit-id: 38a9a7c779e6b5552cfe6a66b7689a306e94d9e2

--- a/.gitmodules
+++ b/.gitmodules
@@ -10,16 +10,25 @@
 [submodule "lib/Requests"]
 	path = lib/Requests
 	url = https://github.com/rmccue/Requests.git
-[submodule "javascripts/flotr2"]
-	path = javascripts/flotr2
+[submodule "js/flotr2"]
+	path = js/flotr2
 	url = https://github.com/HumbleSoftware/Flotr2.git
 [submodule "lib/phpquery"]
 	path = lib/phpquery
 	url = https://github.com/TobiaszCudnik/phpquery.git
-[submodule "javascripts/sigma"]
-	path = javascripts/sigma
+[submodule "js/sigma"]
+	path = js/sigma
 	url = https://github.com/jacomyal/sigma.js.git
-[submodule "javascripts/bubbletree"]
-	path = javascripts/bubbletree
+[submodule "js/bubbletree"]
+	path = js/bubbletree
 	url = https://github.com/okfn/bubbletree.git
+[submodule "lib/querypath"]
+	path = lib/querypath
+	url = https://github.com/technosophos/querypath.git
+[submodule "lib/amon-php"]
+	path = lib/amon-php
+	url = https://github.com/martinrusev/amon-php.git
+[submodule "documents/lib/parsedatetime"]
+	path = documents/lib/parsedatetime
+	url = git://github.com/bear/parsedatetime.git
 

file:a/about.php -> file:b/about.php
--- a/about.php
+++ b/about.php
@@ -1,17 +1,19 @@
 <?php
 include_once('include/common.inc.php');
-include_header();
+include_header('About');
 ?>
 <div class="foundation-header">
     <h1><a href="about.php">About/FAQ</a></h1>
     <h4 class="subheader">Lorem ipsum.</h4>
 </div>
 <h2> What is this? </h2>
-Disclosr is a project to monitor Australian Federal Government agencies 
-compliance with their <a href="http://www.oaic.gov.au/publications/other_operational/foi_policy_frequently_asked_questions.html#_Toc291837571">"proactive disclosure requirements"</a>.
-OGRE (Open Government Realization Evaluation) is a ranking of compliance with these requirements.
-Prometheus is the agent which polls agency websites to assess compliance.
-
+Disclo.gs is a project to monitor Australian Federal Government agencies 
+compliance with their <a href="http://www.oaic.gov.au/publications/other_operational/foi_policy_frequently_asked_questions.html#_Toc291837571">"proactive disclosure requirements" to make a transparency league table as suggested by gov2 taskforce http://gov2.net.au/blog/2009/09/19/a-league-ladder-of-psi-openness/</a>.
+<h2> Attributions </h2>
+National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm <br/>
+data.gov.au http://data.gov.au/dataset/directory-gov-au-full-data-export/ <br/>
+directory.gov.au <br/>
+australia.gov.au http://australia.gov.au/about/copyright <br/>
 <h2> Open everything </h2>
 All documents released CC-BY 3 AU
 Open source git @

file:b/admin/agls.php (new)
--- /dev/null
+++ b/admin/agls.php
@@ -1,1 +1,35 @@
+<?php
 
+include_once('include/common.inc.php');
+include_header('Webserver and Accessiblity');
+
+echo "<table>
+    <tr><th>name</th><th>function</th></tr>";
+$db = $server->get_db('disclosr-agencies');
+try {
+     $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+
+
+    if ($agencies) {
+        foreach ($agencies as $row) {
+
+            echo "<tr><td>" . $row->value->name . "</td>";
+            if (isset($row->value->metaTags)) {
+                if (is_array($row->value->metaTags)) {
+                        $tags =$row->value->metaTags;
+                } else {
+			$tags = object_to_array($row->value->metaTags);
+                }
+                if (isset($tags['AGLS.Function'])) {
+                    echo "<td>" . $tags['AGLS.Function'] . "</td>";
+                }
+            }
+            echo "</tr>";
+        }
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+include_footer();
+?>
+

--- a/admin/conflicts.php
+++ b/admin/conflicts.php
@@ -1,7 +1,7 @@
 <?php
 
 include_once('../include/common.inc.php');
-include_header();
+include_header('Fix Conflicts');
                 require_once '../lib/php-diff/lib/Diff.php';
                 require_once '../lib/php-diff/lib/Diff/Renderer/Html/SideBySide.php';
 

--- a/admin/exportEmployees.csv.php
+++ b/admin/exportEmployees.csv.php
@@ -5,7 +5,6 @@
 $format = "csv";
 //$format = "json";
 if (isset($_REQUEST['format']))  $format = $_REQUEST['format'];
-
 setlocale(LC_CTYPE, 'C');
 if ($format == "csv") {
     $headers = Array("name");

--- /dev/null
+++ b/admin/genericAgencyFixer.php
@@ -1,1 +1,44 @@
+<?php
 
+include_once("../include/common.inc.php");
+require($basePath . 'lib/phpquery/phpQuery/phpQuery.php');
+
+setlocale(LC_CTYPE, 'C');
+
+
+$db = $server->get_db('disclosr-agencies');
+
+try {
+    $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
+    //print_r($rows);
+    foreach ($agencies as $agency) {
+        //echo $agency->value->name . " ".$agency->value->website."<br />\n";
+         // print_r($agency);
+        //hasRestricitiveLicence"	hasRestrictiveLicense -> has Restrictive Licence
+        // "hasYoutube" -> Tube
+        // "comment" -> "comments"
+        if (!isset($agency->value->metaTags) && isset($agency->value->website)) {
+                echo $agency->value->name . " ".$agency->value->website."<br />\n";
+            $agency->value->metaTags = Array();
+            $request = Requests::get($agency->value->website);
+            $html = phpQuery::newDocumentHTML($request->body);
+            phpQuery::selectDocument($html);
+            foreach (pq('meta')->elements as $meta) {
+                $tagName = $meta->getAttribute('name');;
+                $content = $meta->getAttribute('content');
+                if ($tagName != "") {
+echo "$tagName == $content <br>\n";
+                 $agency->value->metaTags[$tagName] = $content;
+                }
+            }
+            //print_r($agency->value->metaTags);
+            $db->save($agency->value);
+            echo "<hr>";
+            flush();
+        }
+    }
+} catch (SetteeRestClientException $e) {
+    setteErrorHandler($e);
+}
+?>
+

--- /dev/null
+++ b/admin/importRTKbodies.php
@@ -1,1 +1,56 @@
+<?php
 
+require_once '../include/common.inc.php';
+
+$db = $server->get_db('disclosr-agencies');
+$rows = $db->get_view("app", "byName")->rows;
+$nametoid = Array();
+$accounts = Array();
+foreach ($rows as $row) {
+    $nametoid[trim($row->key)] = $row->value;
+}
+
+function extractCSVAccounts($url, $nameField, $accountField, $filter) {
+    global $accounts, $nametoid;
+    $request = Requests::get($url);
+    echo $url;
+    $Data = str_getcsv($request->body, "\n"); //parse the rows 
+    $headers = Array();
+    foreach ($Data as $num => $line) {
+        $Row = str_getcsv($line, ",");
+        if ($num == 0) {
+            $headers = $Row;
+            print_r($headers);
+        } else {
+            if (isset($Row[array_search($nameField, $headers)])) {
+                $agencyName = $Row[array_search($nameField, $headers)];
+                    if (!in_array(trim($agencyName), array_keys($nametoid))) {
+                        echo "$agencyName missing" . PHP_EOL;
+                    } else {
+                        echo $Row[array_search($nameField, $headers)] . PHP_EOL;
+                             $accounts[$nametoid[trim($agencyName)]]["rtkURLs"][$agencyName] = 'http://www.righttoknow.org.au/body/'.$Row[array_search($accountField, $headers)];
+                    }
+               } else {
+                echo "error finding any agency" . $line . PHP_EOL;
+            }
+        }
+    }
+}
+
+extractCSVAccounts("http://www.righttoknow.org.au/body/all-authorities.csv","Agency","URL name");
+print_r($accounts);
+/* foreach ($accounts as $id => $accountTypes) {
+    echo $id . "<br>" . PHP_EOL;
+    $doc = object_to_array($db->get($id));
+    // print_r($doc);
+
+    foreach ($accountTypes as $accountType => $accounts) {
+        if (!isset($doc["has" . $accountType]) || !is_array($doc["has" . $accountType])) {
+            $doc["has" . $accountType] = Array();
+        }
+        $doc["has" . $accountType] = array_unique(array_merge($doc["has" . $accountType], $accounts));
+    }
+    $db->save($doc);
+}*/
+?>
+

--- /dev/null
+++ b/admin/naa-agift-tree.php
@@ -1,1 +1,2504 @@
 
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+    "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+    <title>Demo of the tree layout in D3.js</title>
+<!-- source: http://blog.pixelingene.com/demos/d3_tree/
+http://javadude.wordpress.com/2012/06/20/d3-js-from-tree-to-cluster-and-radial-projection/
+http://bl.ocks.org/1249394-->
+    <style>
+        g.node {
+            font-family: Verdana, Helvetica;
+            font-size: 12px;
+            font-weight: bold;
+        }
+        circle.node-dot {
+            fill: lightsalmon;
+            stroke: red;
+            stroke-width: 1px;
+        }
+
+        path.link {
+            fill: none;
+            stroke: gray;
+        }
+        
+    </style>
+</head>
+<body>
+
+<div id="tree-container"></div>
+ <div id="viz"></div>
+
+<script type="text/javascript" src="../js/jquery-1.6.2.min.js"></script>
+<script type="text/javascript" src="../js/d3.min.js"></script>
+<script type="text/javascript" src="../js/d3.layout.min.js"></script>
+
+<script>
+    /**
+ * Created by JetBrains RubyMine.
+ * User: pavanpodila
+ * Date: 7/17/11
+ * Time: 4:30 PM
+ * To change this template use File | Settings | File Templates.
+ */
+
+var treeData = 
+    <?php
+   //     $json = file_get_children('naa-agift.json');
+   // $raw = json_decode($json);
+   // echo $json;
+   // print_r($raw);
+    $raw = json_decode('[
+   "Home",
+   "index.htm",
+   [
+      "BUSINESS SUPPORT AND REGULATION",
+      "000411.htm",
+      [
+         "Association registration",
+         "000396.htm"
+      ],
+      [
+         "Business process auditing",
+         "000413.htm"
+      ],
+      [
+         "Business registration and licensing",
+         "000414.htm"
+      ],
+      [
+         "Business sponsorship",
+         "001371.htm"
+      ],
+      [
+         "Consumer protection",
+         "000439.htm"
+      ],
+      [
+         "Fair trading compliance",
+         "000482.htm",
+         [
+            "Prices surveillance",
+            "000533.htm"
+         ],
+         [
+            "Product safety",
+            "000534.htm"
+         ],
+         [
+            "Trade practices compliance",
+            "000583.htm"
+         ]
+      ],
+      [
+         "Financial institutions regulation",
+         "000485.htm"
+      ],
+      [
+         "Government procurement regulation",
+         "000491.htm"
+      ],
+      [
+         "Industry assistance schemes",
+         "000499.htm"
+      ],
+      [
+         "Industry development",
+         "000500.htm"
+      ],
+      [
+         "Insurance regulation",
+         "000501.htm",
+         [
+            "General insurance",
+            "000489.htm"
+         ],
+         [
+            "Life insurance",
+            "000505.htm"
+         ]
+      ],
+      [
+         "Professional accreditation",
+         "000535.htm"
+      ],
+      [
+         "Small business services",
+         "000565.htm",
+         [
+            "Small business advocacy",
+            "000563.htm"
+         ],
+         [
+            "Small business development",
+            "000564.htm"
+         ]
+      ],
+      [
+         "Stock market regulation",
+         "000572.htm"
+      ]
+   ],
+   [
+      "CIVIC INFRASTRUCTURE",
+      "000424.htm",
+      [
+         "Civic management",
+         "000425.htm",
+         [
+            "Architectural services",
+            "000387.htm"
+         ],
+         [
+            "Building approval services",
+            "000406.htm"
+         ],
+         [
+            "Building regulations and standards",
+            "000408.htm"
+         ],
+         [
+            "Engineering services",
+            "000477.htm"
+         ],
+         [
+            "Town planning",
+            "000582.htm"
+         ]
+      ],
+      [
+         "Energy supply",
+         "000476.htm"
+      ],
+      [
+         "Integrated services planning",
+         "000503.htm"
+      ],
+      [
+         "Public housing",
+         "000536.htm",
+         [
+            "Public housing construction",
+            "000537.htm"
+         ],
+         [
+            "Public housing design",
+            "000538.htm"
+         ],
+         [
+            "Public housing maintenance",
+            "000540.htm"
+         ]
+      ],
+      [
+         "Public land management",
+         "000541.htm",
+         [
+            "Burial ground management",
+            "000410.htm"
+         ],
+         [
+            "Cultural centre management",
+            "000444.htm"
+         ],
+         [
+            "Garden management",
+            "000488.htm"
+         ],
+         [
+            "Memorial maintenance",
+            "000509.htm"
+         ],
+         [
+            "Recreational park management",
+            "000549.htm"
+         ],
+         [
+            "Sporting facilities management",
+            "000571.htm"
+         ]
+      ],
+      [
+         "Regional development",
+         "000552.htm"
+      ],
+      [
+         "Transport network maintenance",
+         "000586.htm"
+      ],
+      [
+         "Waste management",
+         "001349.htm"
+      ]
+   ],
+   [
+      "COMMUNICATIONS",
+      "000433.htm",
+      [
+         "Advertising standards",
+         "000381.htm"
+      ],
+      [
+         "Broadcasting",
+         "000403.htm",
+         [
+            "Broadcasting standards",
+            "000404.htm"
+         ],
+         [
+            "Radio broadcasting",
+            "000546.htm"
+         ],
+         [
+            "Television broadcasting",
+            "000580.htm"
+         ]
+      ],
+      [
+         "Call centre administration",
+         "000417.htm"
+      ],
+      [
+         "Electronic commerce",
+         "000468.htm",
+         [
+            "Authentication",
+            "000399.htm"
+         ],
+         [
+            "Online transaction standards",
+            "000526.htm"
+         ]
+      ],
+      [
+         "Government media",
+         "000490.htm"
+      ],
+      [
+         "Information management standards",
+         "000283.htm",
+         [
+            "Data management",
+            "000448.htm"
+         ],
+         [
+            "Information dissemination",
+            "000502.htm"
+         ],
+         [
+            "Information technology standards",
+            "000282.htm"
+         ]
+      ],
+      [
+         "Media ownership regulation",
+         "000508.htm"
+      ],
+      [
+         "Postal services",
+         "000530.htm",
+         [
+            "Courier services",
+            "000441.htm"
+         ],
+         [
+            "Electronic postal services",
+            "000469.htm"
+         ],
+         [
+            "Retail postal services",
+            "000554.htm"
+         ]
+      ],
+      [
+         "Publishing",
+         "000543.htm",
+         [
+            "Electronic publishing",
+            "000470.htm"
+         ],
+         [
+            "Publishing standards",
+            "000544.htm"
+         ],
+         [
+            "Website development",
+            "000591.htm"
+         ]
+      ],
+      [
+         "Radio communication",
+         "000547.htm",
+         [
+            "Apparatus licensing",
+            "000385.htm"
+         ],
+         [
+            "Spectrum management",
+            "000570.htm"
+         ]
+      ],
+      [
+         "Satellite communication",
+         "000560.htm"
+      ],
+      [
+         "Telecommunications",
+         "000578.htm",
+         [
+            "Carriage service providers",
+            "000420.htm"
+         ],
+         [
+            "Carrier licensing",
+            "000421.htm"
+         ],
+         [
+            "Equipment licensing",
+            "000480.htm"
+         ],
+         [
+            "Mobile telephone services",
+            "000516.htm"
+         ],
+         [
+            "Telephone services",
+            "000579.htm"
+         ]
+      ]
+   ],
+   [
+      "COMMUNITY SERVICES",
+      "000435.htm",
+      [
+         "Accommodation services",
+         "000377.htm",
+         [
+            "Defence housing",
+            "000458.htm"
+         ],
+         [
+            "Emergency accommodation",
+            "000471.htm"
+         ],
+         [
+            "Public housing entitlements",
+            "000539.htm"
+         ],
+         [
+            "Refuge support",
+            "000551.htm"
+         ]
+      ],
+      [
+         "Community support",
+         "000436.htm",
+         [
+            "Adoption services",
+            "000378.htm"
+         ],
+         [
+            "Aged care services",
+            "000382.htm"
+         ],
+         [
+            "Child and youth support",
+            "000422.htm"
+         ],
+         [
+            "Child-care services",
+            "000423.htm"
+         ],
+         [
+            "Defence community programs",
+            "000452.htm"
+         ],
+         [
+            "Family reunion programs",
+            "000483.htm"
+         ],
+         [
+            "Veterans\" entitlements",
+            "000587.htm"
+         ]
+      ],
+      [
+         "Counselling services",
+         "000440.htm"
+      ],
+      [
+         "Emergency services",
+         "000474.htm",
+         [
+            "Ambulance services",
+            "000384.htm"
+         ],
+         [
+            "Emergency funding",
+            "000472.htm"
+         ],
+         [
+            "Firefighting services",
+            "000487.htm"
+         ]
+      ],
+      [
+         "Financial assistance",
+         "000484.htm",
+         [
+            "Benefits",
+            "000402.htm"
+         ],
+         [
+            "Income support schemes",
+            "000494.htm"
+         ]
+      ],
+      [
+         "Natural disasters",
+         "000521.htm",
+         [
+            "Disaster recovery",
+            "000462.htm"
+         ],
+         [
+            "Disaster relief",
+            "000463.htm"
+         ]
+      ],
+      [
+         "Rural community development",
+         "000557.htm"
+      ],
+      [
+         "Social justice and equity",
+         "000566.htm"
+      ],
+      [
+         "Transport access schemes",
+         "000901.htm"
+      ]
+   ],
+   [
+      "CULTURAL AFFAIRS",
+      "000442.htm",
+      [
+         "Arts development",
+         "000391.htm",
+         [
+            "Arts funding",
+            "000393.htm"
+         ],
+         [
+            "Arts incentive schemes",
+            "000394.htm"
+         ],
+         [
+            "Arts promotion",
+            "000395.htm"
+         ]
+      ],
+      [
+         "Collection access",
+         "000427.htm",
+         [
+            "Descriptive standards",
+            "000461.htm"
+         ],
+         [
+            "Finding aids development",
+            "000486.htm"
+         ],
+         [
+            "Reference services",
+            "000550.htm"
+         ]
+      ],
+      [
+         "Collection management",
+         "000430.htm",
+         [
+            "Artefact export regulation",
+            "000390.htm"
+         ],
+         [
+            "Collection accessioning",
+            "000428.htm"
+         ],
+         [
+            "Collection acquisition",
+            "000429.htm"
+         ],
+         [
+            "Collection storage",
+            "000432.htm"
+         ],
+         [
+            "Preservation services",
+            "000532.htm"
+         ]
+      ],
+      [
+         "Collection promotion",
+         "000431.htm",
+         [
+            "Exhibition programs",
+            "000481.htm"
+         ],
+         [
+            "Publicity programs",
+            "000542.htm"
+         ]
+      ],
+      [
+         "Cultural awards and scholarships",
+         "000443.htm"
+      ],
+      [
+         "Cultural festivals",
+         "000445.htm"
+      ],
+      [
+         "Cultural gifts programs",
+         "000446.htm"
+      ],
+      [
+         "Multicultural heritage promotion",
+         "000518.htm",
+         [
+            "Multicultural festivals",
+            "000517.htm"
+         ],
+         [
+            "Multicultural services",
+            "000519.htm"
+         ]
+      ]
+   ],
+   [
+      "DEFENCE",
+      "000449.htm",
+      [
+         "Australian Defence Forces",
+         "000397.htm",
+         [
+            "Air Force",
+            "002266.htm"
+         ],
+         [
+            "Army",
+            "000388.htm"
+         ],
+         [
+            "Badges and insignia",
+            "000401.htm"
+         ],
+         [
+            "Cadets",
+            "000416.htm"
+         ],
+         [
+            "Defence force commands",
+            "000456.htm"
+         ],
+         [
+            "Military bands",
+            "000510.htm"
+         ],
+         [
+            "Navy",
+            "000524.htm"
+         ],
+         [
+            "Reserves",
+            "000553.htm"
+         ],
+         [
+            "Roulettes",
+            "000556.htm"
+         ]
+      ],
+      [
+         "Defence estate management",
+         "000454.htm"
+      ],
+      [
+         "Defence force careers",
+         "000455.htm",
+         [
+            "Defence career development",
+            "000450.htm"
+         ],
+         [
+            "Military employment services",
+            "000512.htm"
+         ]
+      ],
+      [
+         "Defence strategic development",
+         "000574.htm",
+         [
+            "Defence efficiency review",
+            "000453.htm"
+         ],
+         [
+            "Defence reform program",
+            "000460.htm"
+         ]
+      ],
+      [
+         "Defence strategic policy",
+         "000576.htm",
+         [
+            "Strategic decisions",
+            "000573.htm"
+         ],
+         [
+            "Strategic planning",
+            "000575.htm"
+         ]
+      ],
+      [
+         "Defence strategic support",
+         "000577.htm",
+         [
+            "Capital equipment programs",
+            "000419.htm"
+         ],
+         [
+            "Defence industry",
+            "000459.htm"
+         ],
+         [
+            "Defence science and technology",
+            "000905.htm"
+         ],
+         [
+            "Logistics",
+            "000506.htm"
+         ]
+      ],
+      [
+         "Emergency management",
+         "000473.htm",
+         [
+            "Civil community assistance",
+            "000426.htm"
+         ],
+         [
+            "Disaster support",
+            "000464.htm"
+         ]
+      ],
+      [
+         "Military law",
+         "001814.htm"
+      ],
+      [
+         "Military operations",
+         "000515.htm",
+         [
+            "Australian theatre of war",
+            "000398.htm"
+         ],
+         [