projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
side by side | plain
Improve OS validation
Improve OS validation

file:a/admin/linkAmendments.php -> file:b/admin/linkAmendments.php 
--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -2,7 +2,7 @@
 include_once ("../lib/common.inc.php");
 $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c  LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
 WHERE
-c."parentCN" > 0 AND p."childCN" IS NULL ';
+c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
 $query = $conn->prepare($query);
 	$query->execute();
 		databaseError($conn->errorInfo());
@@ -28,9 +28,9 @@
       FROM contractnotice
       GROUP BY "parentCN" 
       HAVING COUNT(*) > 1 
-      AND "parentCN" != 0
+      AND "parentCN" IS NOT NULL
 )
-AND "childCN" = 0
+AND "childCN" IS NULL
 GROUP BY "parentCN" having count(*) > 1';
 $query = $conn->prepare($query);
 	$query->execute();
file:a/admin/partialdata/scraper.php (deleted) 
--- a/admin/partialdata/scraper.php
+++ /dev/null
@@ -1,74 +1,1 @@
-<?php
-date_default_timezone_set('Australia/Melbourne');
-$split = false;
-function format_bytes($size) {
-    $units = array(' B', ' KB', ' MB', ' GB', ' TB');
-    for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
-    return round($size, 2).$units[$i];
-}
 
-$days = 4;
-if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
-$startDate = strtotime("05-Jun-2008");
-if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
-
-function getFile($startDate, $days, $minVal, $maxVal) {
-global $split;
-	$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
-$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
-echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
-$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
-echo "<!-- $url -->";
-$current = file_get_contents($url);
-if (strpos($current,"There are no results that match your selection.")> 0 ) { 
- echo "<font color=red>Empty file!</font><br>";
-}
-if (strpos($current,"Your search returned more than 1000 results.") === false) {
-	file_put_contents($file, $current);
-	echo "$file saved<br>";
-	echo format_bytes(filesize($file))."<br>";
-	echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
-		echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
-	echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
-	flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Success so fetching next $days... <br>";
-getFile($endDate, $days, "" , "");
-}
-	return true;
-} else  {
-	echo "<font color=red>Too many records!</font><br>";
-	echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
-		echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
-	flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Failure so splitting ... <br>";
- doSplit($startDate, $days);
-}
-	return false;
-}
-}
-function doSplit($startDate, $days) {
-global $split;
-$split = true;
-set_time_limit(20);
-getFile($startDate, $days, 0, 12000);
-getFile($startDate, $days, 12000, 16000);
- getFile($startDate, $days, 16000, 20000);
- getFile($startDate, $days, 20000, 30000);
- getFile($startDate, $days, 30000, 40000);
-// getFile($startDate, $days, 40000, 80000);
- getFile($startDate, $days, 40000, 60000);
- getFile($startDate, $days, 60000, 80000);
-// getFile($startDate, $days, 80000, 300000);
- getFile($startDate, $days, 80000, 150000);
- getFile($startDate, $days, 150000, 300000);
- getFile($startDate, $days, 300000, 999999999);
-}
-if (isset($_REQUEST['split'])) {
-	doSplit($startDate, $days);
-} else {
-	getFile($startDate, $days, "" , "");
-}
-?>
-
file:a/admin/updateUNSPSC.php -> file:b/admin/updateUNSPSC.php 
--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -43,6 +43,10 @@
     $unspsc[$tobacco] = $row['UNSPSC'];

     $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);

     $unspsc[$architect] = $row['UNSPSC'];

+        $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']);

+    $unspsc[$powercable] = $row['UNSPSC'];

+        $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);

+    $unspsc[$architect] = $row['UNSPSC'];

     // some just plain wrong

     $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);

     $unspsc[$noOilRigs] = $row['UNSPSC'];
file:a/displayBubbletree.php -> file:b/displayBubbletree.php 
--- a/displayBubbletree.php
+++ b/displayBubbletree.php
@@ -32,48 +32,48 @@
 }
 $total = 0;
 
-$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat 
+$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat 
 FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;');
 $catsresult->execute();
 foreach ($catsresult->fetchAll() as $row) {
 	$cats[] = $row['cat'];
 }
-$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat
+$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat
 FROM contractnotice
 WHERE "categoryUNSPSC" IS NOT NULL
 GROUP BY  "categoryUNSPSC"  order by cat ;');
 $cattwosresult->execute();
 		foreach ($cattwosresult->fetchAll() as $tworow) { 
-			$cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat'];
+			$cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat'];
 		}
 				
 	
 
-$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value
+$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value
 FROM contractnotice
 WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL
 GROUP BY cat order by cat ;');
 $catthreesresult->execute();
 		foreach ($catthreesresult->fetchAll() as $threerow) { 
-			$catthrees[substr($threerow['cat'],0,2)][] = $threerow;
+			$catthrees[substr($threerow['cat'],0,4)][] = $threerow;
 		}
 $nodes = Array();
 foreach ($cats as $catNum) {
 $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5));
-	$catName = $unspsc[$catNum . "0000000"] . $catNum;
+	$catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum;
 	$subnodes = Array();
 	$catValue = 0;
 	foreach ($cattwos[$catNum] as $twoCatNum) {
-		$subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum;
+		$subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum;
 		$subsubnodes = Array();
-	$subCatValue = 0;
+	$subCatValue = 1;
 		$subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5));
 		foreach ($catthrees[$twoCatNum] as $threerow) {
-			$subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat'];
+			$subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat'];
 			$subsubnodes[] = Array(
 				"label" => $subsubcatName,
 				"amount" => $threerow['value'],
-				//"color" => "#".$subCatColor
+				"color" => "#".$subCatColor
 			);
 			$subCatValue += $threerow['value'];
 		}
file:a/exportData.csv.php -> file:b/exportData.csv.php 
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -3,17 +3,31 @@
 include_once("./lib/common.inc.php");
 setlocale(LC_CTYPE, 'C');
 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
+
+$unspsc = Array();
+$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';');
+$unspscresult->execute();
+foreach ($unspscresult->fetchAll() as $row) {
+    $unspsc[$row['UNSPSC']] = $row['Title'];
+}
+
 $query = $conn->prepare('
-SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", 
+SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",
+EXTRACT(EPOCH FROM "publishDate") as "publishDate",
+EXTRACT(EPOCH FROM "contractStart") as "contractStart",
+EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",
+value,description,"procurementMethod",category as cat3,
+(substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "cat1",  
+(substr( "categoryUNSPSC"::text, 0, 5 ) || \'0000\'::text) as "cat2",
  "supplierABN","supplierName",
 (
  case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
  
 (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL 
-FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" is null'
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName"  
+where "childCN" is null'
         , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
 // "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
-// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3")
 $query->execute();
 $errors = $conn->errorInfo();
 if ($errors[2] != "") {
@@ -21,8 +35,8 @@
 }
 
 $num_fields = $query->columnCount();
-$headers = array();
-for ($i = 0; $i < $num_fields; $i++) {
+$headers = Array();
+for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header
     $meta = $query->getColumnMeta($i);
     $headers[] = $meta['name'];
 }
@@ -41,6 +55,13 @@
                     || $headers[$key] == "contractEnd") {
                 $colvalue = date("Y-m-d", $colvalue);
             }
+            if ($headers[$key] == "CNID") {
+                $colvalue = str_replace("A","", $colvalue);
+}
+            if ($headers[$key] == "cat1" || $headers[$key] == "cat2"
+                    || $headers[$key] == "cat3") {
+                $colvalue = $unspsc[$colvalue];
+            }
         }
         fputcsv($fp, array_values($row));
     }
file:b/robots.txt (new) 
--- /dev/null
+++ b/robots.txt
@@ -1,1 +1,3 @@
+User-agent: *
+Disallow: /admin
Unnamed repository; edit this file to name it for gitweb.
GitPHP by Chris Han