<?php | <?php |
include_once ("../lib/common.inc.php"); | include_once ("../lib/common.inc.php"); |
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" | $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" |
WHERE | WHERE |
c."parentCN" > 0 AND p."childCN" IS NULL '; | c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; |
$query = $conn->prepare($query); | $query = $conn->prepare($query); |
$query->execute(); | $query->execute(); |
databaseError($conn->errorInfo()); | databaseError($conn->errorInfo()); |
foreach ($query->fetchAll() as $row) { | foreach ($query->fetchAll() as $row) { |
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . | $conn->exec('UPDATE contractnotice SET "childCN" = \'' . |
$row['CNID'] . '\' where "CNID" = \'' . | $row['CNID'] . '\' where "CNID" = \'' . |
$row['parentCN'] . '\';'); | $row['parentCN'] . '\';'); |
echo 'UPDATE contractnotice SET "childCN" = \'' . | echo 'UPDATE contractnotice SET "childCN" = \'' . |
$row['CNID'] . '\' where "CNID" = \'' . | $row['CNID'] . '\' where "CNID" = \'' . |
$row['parentCN'] . '\';'; | $row['parentCN'] . '\';'; |
$errors = $conn->errorInfo(); | $errors = $conn->errorInfo(); |
if ($errors[1] == 7 || $errors[1] ==0) | if ($errors[1] == 7 || $errors[1] ==0) |
echo $row['CNID'] . " linked to parent " . $row['parentCN'] . | echo $row['CNID'] . " linked to parent " . $row['parentCN'] . |
"<br>\n"; | "<br>\n"; |
else print_r($errors); | else print_r($errors); |
} | } |
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 | // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 |
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN | $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN |
( | ( |
SELECT "parentCN" | SELECT "parentCN" |
FROM contractnotice | FROM contractnotice |
GROUP BY "parentCN" | GROUP BY "parentCN" |
HAVING COUNT(*) > 1 | HAVING COUNT(*) > 1 |
AND "parentCN" != 0 | AND "parentCN" IS NOT NULL |
) | ) |
AND "childCN" = 0 | AND "childCN" IS NULL |
GROUP BY "parentCN" having count(*) > 1'; | GROUP BY "parentCN" having count(*) > 1'; |
$query = $conn->prepare($query); | $query = $conn->prepare($query); |
$query->execute(); | $query->execute(); |
databaseError($conn->errorInfo()); | databaseError($conn->errorInfo()); |
foreach ($query->fetchAll() as $row) { | foreach ($query->fetchAll() as $row) { |
$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg'])); | $cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg'])); |
$last_cnid = array_pop($cnids); | $last_cnid = array_pop($cnids); |
foreach ($cnids as $cnid) { | foreach ($cnids as $cnid) { |
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . | $conn->exec('UPDATE contractnotice SET "childCN" = \'' . |
$last_cnid . '\' where "CNID" = \'' . | $last_cnid . '\' where "CNID" = \'' . |
$cnid . '\';'); | $cnid . '\';'); |
$errors = $conn->errorInfo(); | $errors = $conn->errorInfo(); |
if ($errors[1] == 7 || $errors[1] ==0) | if ($errors[1] == 7 || $errors[1] ==0) |
echo $cnid . " linked to latest child " . $last_cnid . | echo $cnid . " linked to latest child " . $last_cnid . |
"<br>\n"; | "<br>\n"; |
else print_r($errors); | else print_r($errors); |
} | } |
} | } |
?> | ?> |
<?php | |
date_default_timezone_set('Australia/Melbourne'); | |
$split = false; | |
function format_bytes($size) { | |
$units = array(' B', ' KB', ' MB', ' GB', ' TB'); | |
for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024; | |
return round($size, 2).$units[$i]; | |
} | |
$days = 4; | |
if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; | |
$startDate = strtotime("05-Jun-2008"); | |
if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; | |
function getFile($startDate, $days, $minVal, $maxVal) { | |
global $split; | |
$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); | |
$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; | |
echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; | |
$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; | |
echo "<!-- $url -->"; | |
$current = file_get_contents($url); | |
if (strpos($current,"There are no results that match your selection.")> 0 ) { | |
echo "<font color=red>Empty file!</font><br>"; | |
} | |
if (strpos($current,"Your search returned more than 1000 results.") === false) { | |
file_put_contents($file, $current); | |
echo "$file saved<br>"; | |
echo format_bytes(filesize($file))."<br>"; | |
echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>'; | |
echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>'; | |
echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>'; | |
flush(); | |
if (!isset($_REQUEST['split']) && !$split) { | |
echo "Success so fetching next $days... <br>"; | |
getFile($endDate, $days, "" , ""); | |
} | |
return true; | |
} else { | |
echo "<font color=red>Too many records!</font><br>"; | |
echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>'; | |
echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>'; | |
flush(); | |
if (!isset($_REQUEST['split']) && !$split) { | |
echo "Failure so splitting ... <br>"; | |
doSplit($startDate, $days); | |
} | |
return false; | |
} | |
} | |
function doSplit($startDate, $days) { | |
global $split; | |
$split = true; | |
set_time_limit(20); | |
getFile($startDate, $days, 0, 12000); | |
getFile($startDate, $days, 12000, 16000); | |
getFile($startDate, $days, 16000, 20000); | |
getFile($startDate, $days, 20000, 30000); | |
getFile($startDate, $days, 30000, 40000); | |
// getFile($startDate, $days, 40000, 80000); | |
getFile($startDate, $days, 40000, 60000); | |
getFile($startDate, $days, 60000, 80000); | |
// getFile($startDate, $days, 80000, 300000); | |
getFile($startDate, $days, 80000, 150000); | |
getFile($startDate, $days, 150000, 300000); | |
getFile($startDate, $days, 300000, 999999999); | |
} | |
if (isset($_REQUEST['split'])) { | |
doSplit($startDate, $days); | |
} else { | |
getFile($startDate, $days, "" , ""); | |
} | |
?> | |
<?php | <?php |
error_reporting(E_ALL); | error_reporting(E_ALL); |
include_once("../lib/common.inc.php"); | include_once("../lib/common.inc.php"); |
$unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); | $unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); |
$unspscresult->execute(); | $unspscresult->execute(); |
foreach ($unspscresult->fetchAll() as $row) { | foreach ($unspscresult->fetchAll() as $row) { |
$unspsc[$row['Title']] = $row['UNSPSC']; | $unspsc[$row['Title']] = $row['UNSPSC']; |
// some Australian spellings | // some Australian spellings |
$isiz = str_replace("iz", "is", $row['Title']); | $isiz = str_replace("iz", "is", $row['Title']); |
$unspsc[$isiz] = $row['UNSPSC']; | $unspsc[$isiz] = $row['UNSPSC']; |
$filfill = str_replace("fill", "fil", $row['Title']); | $filfill = str_replace("fill", "fil", $row['Title']); |
$unspsc[$filfill] = $row['UNSPSC']; | $unspsc[$filfill] = $row['UNSPSC']; |
$defence = str_replace("efense", "efence", $row['Title']); | $defence = str_replace("efense", "efence", $row['Title']); |
$unspsc[$defence] = $row['UNSPSC']; | $unspsc[$defence] = $row['UNSPSC']; |
$armor = str_replace("rmored", "rmoured", $row['Title']); | $armor = str_replace("rmored", "rmoured", $row['Title']); |
$unspsc[$armor] = $row['UNSPSC']; | $unspsc[$armor] = $row['UNSPSC']; |
$erre = str_replace("er", "re", $row['Title']); | $erre = str_replace("er", "re", $row['Title']); |
$unspsc[$erre] = $row['UNSPSC']; | $unspsc[$erre] = $row['UNSPSC']; |
$center = str_replace("center", "centre", $row['Title']); | $center = str_replace("center", "centre", $row['Title']); |
$unspsc[$center] = $row['UNSPSC']; | $unspsc[$center] = $row['UNSPSC']; |
$accessory = str_replace("accesor", "accessor", $row['Title']); | $accessory = str_replace("accesor", "accessor", $row['Title']); |
$unspsc[$accessory] = $row['UNSPSC']; | $unspsc[$accessory] = $row['UNSPSC']; |
$lyslyz = str_replace("lyz", "lys", $row['Title']); | $lyslyz = str_replace("lyz", "lys", $row['Title']); |
$unspsc[$lyslyz] = $row['UNSPSC']; | $unspsc[$lyslyz] = $row['UNSPSC']; |
$tire = str_replace("ire", "yre", $row['Title']); | $tire = str_replace("ire", "yre", $row['Title']); |
$unspsc[$tire] = $row['UNSPSC']; | $unspsc[$tire] = $row['UNSPSC']; |
$pe = str_replace("pe", "pae", $row['Title']); | $pe = str_replace("pe", "pae", $row['Title']); |
$unspsc[$pe] = $row['UNSPSC']; | $unspsc[$pe] = $row['UNSPSC']; |
$ane = str_replace("ane", "anae", $row['Title']); | $ane = str_replace("ane", "anae", $row['Title']); |
$unspsc[$ane] = $row['UNSPSC']; | $unspsc[$ane] = $row['UNSPSC']; |
$airo = str_replace("airplane", "aeroplane", $row['Title']); | $airo = str_replace("airplane", "aeroplane", $row['Title']); |
$unspsc[$airo] = $row['UNSPSC']; | $unspsc[$airo] = $row['UNSPSC']; |
// some divergence from standard | // some divergence from standard |
$forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']); | $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']); |
$unspsc[$forensicit] = $row['UNSPSC']; | $unspsc[$forensicit] = $row['UNSPSC']; |
$powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']); | $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']); |
$unspsc[$powercable] = $row['UNSPSC']; | $unspsc[$powercable] = $row['UNSPSC']; |
$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); | $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); |
$unspsc[$tobacco] = $row['UNSPSC']; | $unspsc[$tobacco] = $row['UNSPSC']; |
$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); | $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); |
$unspsc[$architect] = $row['UNSPSC']; | $unspsc[$architect] = $row['UNSPSC']; |
$powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']); | |
$unspsc[$powercable] = $row['UNSPSC']; | |
$forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); | |
$unspsc[$architect] = $row['UNSPSC']; | |
// some just plain wrong | // some just plain wrong |
$noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); | $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); |
$unspsc[$noOilRigs] = $row['UNSPSC']; | $unspsc[$noOilRigs] = $row['UNSPSC']; |
} | } |
$query = 'SELECT "CNID","category","value" | $query = 'SELECT "CNID","category","value" |
FROM contractnotice | FROM contractnotice |
WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; | WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; |
$emptycatresult = $conn->prepare($query); | $emptycatresult = $conn->prepare($query); |
$missing = Array(); | $missing = Array(); |
$emptycatresult->execute(); | $emptycatresult->execute(); |
foreach ($emptycatresult->fetchAll() as $record) { | foreach ($emptycatresult->fetchAll() as $record) { |
if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") { | if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") { |
$missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value']; | $missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value']; |
// echo "<br>\n Category not found for: \n"; | // echo "<br>\n Category not found for: \n"; |
// print_r($record); | // print_r($record); |
} else { | } else { |
$result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" = | $result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" = |
\'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';'); | \'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';'); |
if ($result) { | if ($result) { |
echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n"; | echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n"; |
} else { | } else { |
echo "error<br>"; | echo "error<br>"; |
print_r($conn->errorInfo()); | print_r($conn->errorInfo()); |
} | } |
} | } |
} | } |
asort($missing, SORT_NUMERIC); | asort($missing, SORT_NUMERIC); |
print_r($missing); | print_r($missing); |
?> | ?> |
<!DOCTYPE html> | <!DOCTYPE html> |
<html xmlns="http://www.w3.org/1999/xhtml"> | <html xmlns="http://www.w3.org/1999/xhtml"> |
<head> | <head> |
<meta charset="UTF-8"/> | <meta charset="UTF-8"/> |
<title>Minimal BubbleTree Demo</title> | <title>Minimal BubbleTree Demo</title> |
<script type="text/javascript" src="lib/bubbletree/lib/jquery-1.5.2.min.js"></script> | <script type="text/javascript" src="lib/bubbletree/lib/jquery-1.5.2.min.js"></script> |
<script type="text/javascript" src="lib/bubbletree/lib/jquery.history.js"></script> | <script type="text/javascript" src="lib/bubbletree/lib/jquery.history.js"></script> |
<script type="text/javascript" src="lib/bubbletree/lib/raphael.js"></script> | <script type="text/javascript" src="lib/bubbletree/lib/raphael.js"></script> |
<script type="text/javascript" src="lib/bubbletree/lib/vis4.js"></script> | <script type="text/javascript" src="lib/bubbletree/lib/vis4.js"></script> |
<script type="text/javascript" src="lib/bubbletree/lib/Tween.js"></script> | <script type="text/javascript" src="lib/bubbletree/lib/Tween.js"></script> |
<script type="text/javascript" src="lib/bubbletree/build/bubbletree.js"></script> | <script type="text/javascript" src="lib/bubbletree/build/bubbletree.js"></script> |
<link rel="stylesheet" type="text/css" href="lib/bubbletree/build/bubbletree.css" /> | <link rel="stylesheet" type="text/css" href="lib/bubbletree/build/bubbletree.css" /> |
<script type="text/javascript" src="lib/bubbletree/styles/cofog.js"></script> | <script type="text/javascript" src="lib/bubbletree/styles/cofog.js"></script> |
<script type="text/javascript"> | <script type="text/javascript"> |
$(function() { | $(function() { |
<?php | <?php |
include_once ("lib/common.inc.php"); | include_once ("lib/common.inc.php"); |
include("lib/Color.php"); | include("lib/Color.php"); |
$color = new Lux_Color(); | $color = new Lux_Color(); |
$unspscresult= $conn->prepare('select * from "UNSPSCcategories";'); | $unspscresult= $conn->prepare('select * from "UNSPSCcategories";'); |
$unspscresult->execute(); | $unspscresult->execute(); |
foreach ($unspscresult->fetchAll() as $row) { | foreach ($unspscresult->fetchAll() as $row) { |
$unspsc[$row['UNSPSC']] = $row['Title']; | $unspsc[$row['UNSPSC']] = $row['Title']; |
} | } |
$total = 0; | $total = 0; |
$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat | $catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat |
FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); | FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); |
$catsresult->execute(); | $catsresult->execute(); |
foreach ($catsresult->fetchAll() as $row) { | foreach ($catsresult->fetchAll() as $row) { |
$cats[] = $row['cat']; | $cats[] = $row['cat']; |
} | } |
$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat | $cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat |
FROM contractnotice | FROM contractnotice |
WHERE "categoryUNSPSC" IS NOT NULL | WHERE "categoryUNSPSC" IS NOT NULL |
GROUP BY "categoryUNSPSC" order by cat ;'); | GROUP BY "categoryUNSPSC" order by cat ;'); |
$cattwosresult->execute(); | $cattwosresult->execute(); |
foreach ($cattwosresult->fetchAll() as $tworow) { | foreach ($cattwosresult->fetchAll() as $tworow) { |
$cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat']; | $cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat']; |
} | } |
$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value | $catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value |
FROM contractnotice | FROM contractnotice |
WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL | WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL |
GROUP BY cat order by cat ;'); | GROUP BY cat order by cat ;'); |
$catthreesresult->execute(); | $catthreesresult->execute(); |
foreach ($catthreesresult->fetchAll() as $threerow) { | foreach ($catthreesresult->fetchAll() as $threerow) { |
$catthrees[substr($threerow['cat'],0,2)][] = $threerow; | $catthrees[substr($threerow['cat'],0,4)][] = $threerow; |
} | } |
$nodes = Array(); | $nodes = Array(); |
foreach ($cats as $catNum) { | foreach ($cats as $catNum) { |
$catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); | $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); |
$catName = $unspsc[$catNum . "0000000"] . $catNum; | $catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum; |
$subnodes = Array(); | $subnodes = Array(); |
$catValue = 0; | $catValue = 0; |
foreach ($cattwos[$catNum] as $twoCatNum) { | foreach ($cattwos[$catNum] as $twoCatNum) { |
$subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum; | $subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum; |
$subsubnodes = Array(); | $subsubnodes = Array(); |
$subCatValue = 0; | $subCatValue = 1; |
$subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); | $subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); |
foreach ($catthrees[$twoCatNum] as $threerow) { | foreach ($catthrees[$twoCatNum] as $threerow) { |
$subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat']; | $subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat']; |
$subsubnodes[] = Array( | $subsubnodes[] = Array( |
"label" => $subsubcatName, | "label" => $subsubcatName, |
"amount" => $threerow['value'], | "amount" => $threerow['value'], |
//"color" => "#".$subCatColor | "color" => "#".$subCatColor |
); | ); |
$subCatValue += $threerow['value']; | $subCatValue += $threerow['value']; |
} | } |
$subnodes[] = Array( | $subnodes[] = Array( |
"label" => $subcatName, | "label" => $subcatName, |
"amount" => $subCatValue, | "amount" => $subCatValue, |
"color" => "#".$subCatColor, | "color" => "#".$subCatColor, |
"children" => $subsubnodes | "children" => $subsubnodes |
); | ); |
$catValue += $subCatValue; | $catValue += $subCatValue; |
} | } |
$nodes[] = Array( | $nodes[] = Array( |
"label" => $catName, | "label" => $catName, |
"amount" => $catValue, | "amount" => $catValue, |
"color" => "#".$catColor, | "color" => "#".$catColor, |
"children" => $subnodes | "children" => $subnodes |
); | ); |
$total += $catValue; | $total += $catValue; |
} | } |
$data = Array( | $data = Array( |
"label" => "Australian Federal Gove |