From: Alexander Sadleir Date: Sat, 10 Dec 2011 05:50:36 +0000 Subject: Improve OS validation X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=5615c5e14c2aeb1bcdeb4553fe5438d023858157 --- Improve OS validation --- --- a/admin/linkAmendments.php +++ b/admin/linkAmendments.php @@ -2,7 +2,7 @@ include_once ("../lib/common.inc.php"); $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" WHERE -c."parentCN" > 0 AND p."childCN" IS NULL '; +c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; $query = $conn->prepare($query); $query->execute(); databaseError($conn->errorInfo()); @@ -28,9 +28,9 @@ FROM contractnotice GROUP BY "parentCN" HAVING COUNT(*) > 1 - AND "parentCN" != 0 + AND "parentCN" IS NOT NULL ) -AND "childCN" = 0 +AND "childCN" IS NULL GROUP BY "parentCN" having count(*) > 1'; $query = $conn->prepare($query); $query->execute(); --- a/admin/partialdata/scraper.php +++ /dev/null @@ -1,74 +1,1 @@ -= 1024 && $i < 4; $i++) $size /= 1024; - return round($size, 2).$units[$i]; -} -$days = 4; -if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; -$startDate = strtotime("05-Jun-2008"); -if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; - -function getFile($startDate, $days, $minVal, $maxVal) { -global $split; - $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); -$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; -echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; -$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; -echo ""; -$current = file_get_contents($url); -if (strpos($current,"There are no results that match your selection.")> 0 ) { - echo "Empty file!
"; -} -if (strpos($current,"Your search returned more than 1000 results.") === false) { - file_put_contents($file, $current); - echo "$file saved
"; - echo format_bytes(filesize($file))."
"; - echo 'Load next '.($days).' days
'; - echo 'Load next '.($days*2).' days
'; - echo 'Load next '.($days).' days with split
'; - flush(); -if (!isset($_REQUEST['split']) && !$split) { -echo "Success so fetching next $days...
"; -getFile($endDate, $days, "" , ""); -} - return true; -} else { - echo "Too many records!
"; - echo 'Load '.($days/2).' days instead?
'; - echo 'Split instead?
'; - flush(); -if (!isset($_REQUEST['split']) && !$split) { -echo "Failure so splitting ...
"; - doSplit($startDate, $days); -} - return false; -} -} -function doSplit($startDate, $days) { -global $split; -$split = true; -set_time_limit(20); -getFile($startDate, $days, 0, 12000); -getFile($startDate, $days, 12000, 16000); - getFile($startDate, $days, 16000, 20000); - getFile($startDate, $days, 20000, 30000); - getFile($startDate, $days, 30000, 40000); -// getFile($startDate, $days, 40000, 80000); - getFile($startDate, $days, 40000, 60000); - getFile($startDate, $days, 60000, 80000); -// getFile($startDate, $days, 80000, 300000); - getFile($startDate, $days, 80000, 150000); - getFile($startDate, $days, 150000, 300000); - getFile($startDate, $days, 300000, 999999999); -} -if (isset($_REQUEST['split'])) { - doSplit($startDate, $days); -} else { - getFile($startDate, $days, "" , ""); -} -?> - --- a/admin/updateUNSPSC.php +++ b/admin/updateUNSPSC.php @@ -43,6 +43,10 @@ $unspsc[$tobacco] = $row['UNSPSC']; $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); $unspsc[$architect] = $row['UNSPSC']; + $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']); + $unspsc[$powercable] = $row['UNSPSC']; + $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); + $unspsc[$architect] = $row['UNSPSC']; // some just plain wrong $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); $unspsc[$noOilRigs] = $row['UNSPSC']; --- a/displayBubbletree.php +++ b/displayBubbletree.php @@ -32,48 +32,48 @@ } $total = 0; -$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat +$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); $catsresult->execute(); foreach ($catsresult->fetchAll() as $row) { $cats[] = $row['cat']; } -$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat +$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL GROUP BY "categoryUNSPSC" order by cat ;'); $cattwosresult->execute(); foreach ($cattwosresult->fetchAll() as $tworow) { - $cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat']; + $cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat']; } -$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value +$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value FROM contractnotice WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL GROUP BY cat order by cat ;'); $catthreesresult->execute(); foreach ($catthreesresult->fetchAll() as $threerow) { - $catthrees[substr($threerow['cat'],0,2)][] = $threerow; + $catthrees[substr($threerow['cat'],0,4)][] = $threerow; } $nodes = Array(); foreach ($cats as $catNum) { $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); - $catName = $unspsc[$catNum . "0000000"] . $catNum; + $catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum; $subnodes = Array(); $catValue = 0; foreach ($cattwos[$catNum] as $twoCatNum) { - $subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum; + $subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum; $subsubnodes = Array(); - $subCatValue = 0; + $subCatValue = 1; $subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); foreach ($catthrees[$twoCatNum] as $threerow) { - $subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat']; + $subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat']; $subsubnodes[] = Array( "label" => $subsubcatName, "amount" => $threerow['value'], - //"color" => "#".$subCatColor + "color" => "#".$subCatColor ); $subCatValue += $threerow['value']; } --- a/exportData.csv.php +++ b/exportData.csv.php @@ -3,17 +3,31 @@ include_once("./lib/common.inc.php"); setlocale(LC_CTYPE, 'C'); // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 + +$unspsc = Array(); +$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';'); +$unspscresult->execute(); +foreach ($unspscresult->fetchAll() as $row) { + $unspsc[$row['UNSPSC']] = $row['Title']; +} + $query = $conn->prepare(' -SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", +SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN", +EXTRACT(EPOCH FROM "publishDate") as "publishDate", +EXTRACT(EPOCH FROM "contractStart") as "contractStart", +EXTRACT(EPOCH FROM "contractEnd") as "contractEnd", +value,description,"procurementMethod",category as cat3, +(substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "cat1", +(substr( "categoryUNSPSC"::text, 0, 5 ) || \'0000\'::text) as "cat2", "supplierABN","supplierName", ( case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL -FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" is null' +FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" +where "childCN" is null' , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); // "supplierCity","supplierPostcode","supplierCountry","contactPostcode", -// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3") $query->execute(); $errors = $conn->errorInfo(); if ($errors[2] != "") { @@ -21,8 +35,8 @@ } $num_fields = $query->columnCount(); -$headers = array(); -for ($i = 0; $i < $num_fields; $i++) { +$headers = Array(); +for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header $meta = $query->getColumnMeta($i); $headers[] = $meta['name']; } @@ -41,6 +55,13 @@ || $headers[$key] == "contractEnd") { $colvalue = date("Y-m-d", $colvalue); } + if ($headers[$key] == "CNID") { + $colvalue = str_replace("A","", $colvalue); +} + if ($headers[$key] == "cat1" || $headers[$key] == "cat2" + || $headers[$key] == "cat3") { + $colvalue = $unspsc[$colvalue]; + } } fputcsv($fp, array_values($row)); } --- /dev/null +++ b/robots.txt @@ -1,1 +1,3 @@ +User-agent: * +Disallow: /admin