From: Alexander Sadleir <maxious@lambdacomplex.org>
Date: Wed, 21 Mar 2012 03:18:54 +0000
Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=6e3b80c17d8ed9f0afe0db0403d9e82652250364
---
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
---


--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -1,6 +1,9 @@
 <?php
 include_once ("../lib/common.inc.php");
 $query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\'';
+$query = $conn->prepare($query);
+	$query->execute();
+        $query = 'update contractnotice set "supplierABN" = null where "supplierABN" = \'0\'';
 $query = $conn->prepare($query);
 	$query->execute();
 $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c  LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"

--- /dev/null
+++ b/admin/partialdata/scraper.txt
@@ -1,1 +1,74 @@
+<?php
+date_default_timezone_set('Australia/Melbourne');
+$split = false;
+function format_bytes($size) {
+    $units = array(' B', ' KB', ' MB', ' GB', ' TB');
+    for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
+    return round($size, 2).$units[$i];
+}
 
+$days = 4;
+if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
+$startDate = strtotime("05-Jun-2008");
+if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
+
+function getFile($startDate, $days, $minVal, $maxVal) {
+global $split;
+	$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
+$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
+echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
+$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
+echo "<!-- $url -->";
+$current = file_get_contents($url);
+if (strpos($current,"There are no results that match your selection.")> 0 ) { 
+ echo "<font color=red>Empty file!</font><br>";
+}
+if (strpos($current,"Your search returned more than 1000 results.") === false) {
+	file_put_contents($file, $current);
+	echo "$file saved<br>";
+	echo format_bytes(filesize($file))."<br>";
+	echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
+		echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
+	echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
+	flush();
+if (!isset($_REQUEST['split']) && !$split) {
+echo "Success so fetching next $days... <br>";
+getFile($endDate, $days, "" , "");
+}
+	return true;
+} else  {
+	echo "<font color=red>Too many records!</font><br>";
+	echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
+		echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
+	flush();
+if (!isset($_REQUEST['split']) && !$split) {
+echo "Failure so splitting ... <br>";
+ doSplit($startDate, $days);
+}
+	return false;
+}
+}
+function doSplit($startDate, $days) {
+global $split;
+$split = true;
+set_time_limit(20);
+getFile($startDate, $days, 0, 12000);
+getFile($startDate, $days, 12000, 16000);
+ getFile($startDate, $days, 16000, 20000);
+ getFile($startDate, $days, 20000, 30000);
+ getFile($startDate, $days, 30000, 40000);
+// getFile($startDate, $days, 40000, 80000);
+ getFile($startDate, $days, 40000, 60000);
+ getFile($startDate, $days, 60000, 80000);
+// getFile($startDate, $days, 80000, 300000);
+ getFile($startDate, $days, 80000, 150000);
+ getFile($startDate, $days, 150000, 300000);
+ getFile($startDate, $days, 300000, 999999999);
+}
+if (isset($_REQUEST['split'])) {
+	doSplit($startDate, $days);
+} else {
+	getFile($startDate, $days, "" , "");
+}
+?>
+

--- a/displaySupplier.php
+++ b/displaySupplier.php
@@ -5,9 +5,9 @@
 
     include_header("Supplier");
     $supplierS = htmlentities(strip_tags($_REQUEST['supplier']));
-    MethodCountGraph($supplierS);
-    CnCGraph($supplierS);
-    MethodValueGraph($supplierS);
+   // MethodCountGraph($supplierS);
+  //  CnCGraph($supplierS);
+   // MethodValueGraph($supplierS);
     /* lobbyist ties
 
       links to ABR/ASIC/Google News/ASX/Court records
@@ -27,12 +27,16 @@
     echo $query;
     $query = $conn->prepare($query);
 
-//$query->bindParam(":supplierName", $supplierName);
+
+  if ($supplierParts[0] > 0) {
     $query->bindParam(":supplierABN", $supplierABN);
+  } else {
+      $query->bindParam(":supplierName", $supplierName);
+  }
     $query->execute();
     databaseError($conn->errorInfo());
-    echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">';
-    echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">';
+   // echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">';
+   // echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">';
 
     echo "<table>  <thead>
     <tr>

--- a/lib/graphs.inc.php
+++ b/lib/graphs.inc.php
@@ -27,7 +27,11 @@
             var d1 = [];
             var d2 = [];
     <?php
-    $query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is null) as a group by cnid order by cnid';
+    $query = 'select cnid, count(*) from 
+        (select ("CNID"::integer - MOD("CNID"::integer,100)) as cnid 
+        from contractnotice where "CNID"::integer < 999999 
+        and "CNID" not like \'%-A%\' 
+        and "parentCN" is null) as a group by cnid order by cnid';
     $query = $conn->prepare($query);
     $query->execute();
     $errors = $conn->errorInfo();
@@ -40,7 +44,9 @@
 
         echo "d1.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n";
     };
-    $query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid';
+    $query = 'select cnid, count(*) from (select ("CNID"::integer - MOD("CNID"::integer,100)) as cnid
+        from contractnotice where "CNID" not like \'%-A%\' and "parentCN" is not null) 
+        as a group by cnid order by cnid';
     $query = $conn->prepare($query);
     $query->execute();
     $errors = $conn->errorInfo();
@@ -71,7 +77,7 @@
                         points: { show: true }
                     },
                     bars: { show: true }
-                },
+                }]
             ];
             var options = 
                 {