Use correct tuples of UNSPSC for export/bubbletree
Use correct tuples of UNSPSC for export/bubbletree

<?php <?php
include_once ("../lib/common.inc.php"); include_once ("../lib/common.inc.php");
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE WHERE
c."parentCN" > 0 AND p."childCN" IS NULL '; c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'); $row['parentCN'] . '\';');
echo 'UPDATE contractnotice SET "childCN" = \'' . echo 'UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'; $row['parentCN'] . '\';';
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $row['CNID'] . " linked to parent " . $row['parentCN'] . echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
   
} }
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN
( (
SELECT "parentCN" SELECT "parentCN"
FROM contractnotice FROM contractnotice
GROUP BY "parentCN" GROUP BY "parentCN"
HAVING COUNT(*) > 1 HAVING COUNT(*) > 1
AND "parentCN" != 0 AND "parentCN" IS NOT NULL
) )
AND "childCN" = 0 AND "childCN" IS NULL
GROUP BY "parentCN" having count(*) > 1'; GROUP BY "parentCN" having count(*) > 1';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg'])); $cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg']));
$last_cnid = array_pop($cnids); $last_cnid = array_pop($cnids);
foreach ($cnids as $cnid) { foreach ($cnids as $cnid) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$last_cnid . '\' where "CNID" = \'' . $last_cnid . '\' where "CNID" = \'' .
$cnid . '\';'); $cnid . '\';');
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $cnid . " linked to latest child " . $last_cnid . echo $cnid . " linked to latest child " . $last_cnid .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
} }
} }
?> ?>
   
<?php <?php
error_reporting(E_ALL); error_reporting(E_ALL);
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
$unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); $unspscresult = $conn->prepare('select * from "UNSPSCcategories";');
$unspscresult->execute(); $unspscresult->execute();
foreach ($unspscresult->fetchAll() as $row) { foreach ($unspscresult->fetchAll() as $row) {
$unspsc[$row['Title']] = $row['UNSPSC']; $unspsc[$row['Title']] = $row['UNSPSC'];
// some Australian spellings // some Australian spellings
$isiz = str_replace("iz", "is", $row['Title']); $isiz = str_replace("iz", "is", $row['Title']);
$unspsc[$isiz] = $row['UNSPSC']; $unspsc[$isiz] = $row['UNSPSC'];
$filfill = str_replace("fill", "fil", $row['Title']); $filfill = str_replace("fill", "fil", $row['Title']);
$unspsc[$filfill] = $row['UNSPSC']; $unspsc[$filfill] = $row['UNSPSC'];
$defence = str_replace("efense", "efence", $row['Title']); $defence = str_replace("efense", "efence", $row['Title']);
$unspsc[$defence] = $row['UNSPSC']; $unspsc[$defence] = $row['UNSPSC'];
$armor = str_replace("rmored", "rmoured", $row['Title']); $armor = str_replace("rmored", "rmoured", $row['Title']);
$unspsc[$armor] = $row['UNSPSC']; $unspsc[$armor] = $row['UNSPSC'];
$erre = str_replace("er", "re", $row['Title']); $erre = str_replace("er", "re", $row['Title']);
$unspsc[$erre] = $row['UNSPSC']; $unspsc[$erre] = $row['UNSPSC'];
$center = str_replace("center", "centre", $row['Title']); $center = str_replace("center", "centre", $row['Title']);
$unspsc[$center] = $row['UNSPSC']; $unspsc[$center] = $row['UNSPSC'];
$accessory = str_replace("accesor", "accessor", $row['Title']); $accessory = str_replace("accesor", "accessor", $row['Title']);
$unspsc[$accessory] = $row['UNSPSC']; $unspsc[$accessory] = $row['UNSPSC'];
$lyslyz = str_replace("lyz", "lys", $row['Title']); $lyslyz = str_replace("lyz", "lys", $row['Title']);
$unspsc[$lyslyz] = $row['UNSPSC']; $unspsc[$lyslyz] = $row['UNSPSC'];
$tire = str_replace("ire", "yre", $row['Title']); $tire = str_replace("ire", "yre", $row['Title']);
$unspsc[$tire] = $row['UNSPSC']; $unspsc[$tire] = $row['UNSPSC'];
$pe = str_replace("pe", "pae", $row['Title']); $pe = str_replace("pe", "pae", $row['Title']);
$unspsc[$pe] = $row['UNSPSC']; $unspsc[$pe] = $row['UNSPSC'];
$ane = str_replace("ane", "anae", $row['Title']); $ane = str_replace("ane", "anae", $row['Title']);
$unspsc[$ane] = $row['UNSPSC']; $unspsc[$ane] = $row['UNSPSC'];
$airo = str_replace("airplane", "aeroplane", $row['Title']); $airo = str_replace("airplane", "aeroplane", $row['Title']);
$unspsc[$airo] = $row['UNSPSC']; $unspsc[$airo] = $row['UNSPSC'];
// some divergence from standard // some divergence from standard
$forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']); $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']);
$unspsc[$forensicit] = $row['UNSPSC']; $unspsc[$forensicit] = $row['UNSPSC'];
$powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']); $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']);
$unspsc[$powercable] = $row['UNSPSC']; $unspsc[$powercable] = $row['UNSPSC'];
$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);
$unspsc[$tobacco] = $row['UNSPSC']; $unspsc[$tobacco] = $row['UNSPSC'];
$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
$unspsc[$architect] = $row['UNSPSC']; $unspsc[$architect] = $row['UNSPSC'];
  $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']);
  $unspsc[$powercable] = $row['UNSPSC'];
  $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
  $unspsc[$architect] = $row['UNSPSC'];
// some just plain wrong // some just plain wrong
$noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);
$unspsc[$noOilRigs] = $row['UNSPSC']; $unspsc[$noOilRigs] = $row['UNSPSC'];
} }
$query = 'SELECT "CNID","category","value" $query = 'SELECT "CNID","category","value"
FROM contractnotice FROM contractnotice
WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0';
$emptycatresult = $conn->prepare($query); $emptycatresult = $conn->prepare($query);
$missing = Array(); $missing = Array();
$emptycatresult->execute(); $emptycatresult->execute();
foreach ($emptycatresult->fetchAll() as $record) { foreach ($emptycatresult->fetchAll() as $record) {
if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") { if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") {
$missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value']; $missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value'];
// echo "<br>\n Category not found for: \n"; // echo "<br>\n Category not found for: \n";
// print_r($record); // print_r($record);
} else { } else {
$result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" = $result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" =
\'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';'); \'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';');
if ($result) { if ($result) {
echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n"; echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n";
} else { } else {
echo "error<br>"; echo "error<br>";
print_r($conn->errorInfo()); print_r($conn->errorInfo());
} }
} }
} }
asort($missing, SORT_NUMERIC); asort($missing, SORT_NUMERIC);
print_r($missing); print_r($missing);
?> ?>
   
   
<!DOCTYPE html> <!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml"> <html xmlns="http://www.w3.org/1999/xhtml">
<head> <head>
<meta charset="UTF-8"/> <meta charset="UTF-8"/>
<title>Minimal BubbleTree Demo</title> <title>Minimal BubbleTree Demo</title>
<script type="text/javascript" src="lib/bubbletree/lib/jquery-1.5.2.min.js"></script> <script type="text/javascript" src="lib/bubbletree/lib/jquery-1.5.2.min.js"></script>
<script type="text/javascript" src="lib/bubbletree/lib/jquery.history.js"></script> <script type="text/javascript" src="lib/bubbletree/lib/jquery.history.js"></script>
<script type="text/javascript" src="lib/bubbletree/lib/raphael.js"></script> <script type="text/javascript" src="lib/bubbletree/lib/raphael.js"></script>
<script type="text/javascript" src="lib/bubbletree/lib/vis4.js"></script> <script type="text/javascript" src="lib/bubbletree/lib/vis4.js"></script>
<script type="text/javascript" src="lib/bubbletree/lib/Tween.js"></script> <script type="text/javascript" src="lib/bubbletree/lib/Tween.js"></script>
<script type="text/javascript" src="lib/bubbletree/build/bubbletree.js"></script> <script type="text/javascript" src="lib/bubbletree/build/bubbletree.js"></script>
<link rel="stylesheet" type="text/css" href="lib/bubbletree/build/bubbletree.css" /> <link rel="stylesheet" type="text/css" href="lib/bubbletree/build/bubbletree.css" />
<script type="text/javascript" src="lib/bubbletree/styles/cofog.js"></script> <script type="text/javascript" src="lib/bubbletree/styles/cofog.js"></script>
<script type="text/javascript"> <script type="text/javascript">
$(function() { $(function() {
<?php <?php
include_once ("lib/common.inc.php"); include_once ("lib/common.inc.php");
   
include("lib/Color.php"); include("lib/Color.php");
$color = new Lux_Color(); $color = new Lux_Color();
   
   
$unspscresult= $conn->prepare('select * from "UNSPSCcategories";'); $unspscresult= $conn->prepare('select * from "UNSPSCcategories";');
$unspscresult->execute(); $unspscresult->execute();
foreach ($unspscresult->fetchAll() as $row) { foreach ($unspscresult->fetchAll() as $row) {
$unspsc[$row['UNSPSC']] = $row['Title']; $unspsc[$row['UNSPSC']] = $row['Title'];
   
} }
$total = 0; $total = 0;
   
$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat $catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat
FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;');
$catsresult->execute(); $catsresult->execute();
foreach ($catsresult->fetchAll() as $row) { foreach ($catsresult->fetchAll() as $row) {
$cats[] = $row['cat']; $cats[] = $row['cat'];
} }
$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat $cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat
FROM contractnotice FROM contractnotice
WHERE "categoryUNSPSC" IS NOT NULL WHERE "categoryUNSPSC" IS NOT NULL
GROUP BY "categoryUNSPSC" order by cat ;'); GROUP BY "categoryUNSPSC" order by cat ;');
$cattwosresult->execute(); $cattwosresult->execute();
foreach ($cattwosresult->fetchAll() as $tworow) { foreach ($cattwosresult->fetchAll() as $tworow) {
$cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat']; $cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat'];
} }
   
$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value $catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value
FROM contractnotice FROM contractnotice
WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL
GROUP BY cat order by cat ;'); GROUP BY cat order by cat ;');
$catthreesresult->execute(); $catthreesresult->execute();
foreach ($catthreesresult->fetchAll() as $threerow) { foreach ($catthreesresult->fetchAll() as $threerow) {
$catthrees[substr($threerow['cat'],0,2)][] = $threerow; $catthrees[substr($threerow['cat'],0,4)][] = $threerow;
} }
$nodes = Array(); $nodes = Array();
foreach ($cats as $catNum) { foreach ($cats as $catNum) {
$catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5));
$catName = $unspsc[$catNum . "0000000"] . $catNum; $catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum;
$subnodes = Array(); $subnodes = Array();
$catValue = 0; $catValue = 0;
foreach ($cattwos[$catNum] as $twoCatNum) { foreach ($cattwos[$catNum] as $twoCatNum) {
$subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum; $subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum;
$subsubnodes = Array(); $subsubnodes = Array();
$subCatValue = 0; $subCatValue = 1;
$subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); $subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5));
foreach ($catthrees[$twoCatNum] as $threerow) { foreach ($catthrees[$twoCatNum] as $threerow) {
$subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat']; $subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat'];
$subsubnodes[] = Array( $subsubnodes[] = Array(
"label" => $subsubcatName, "label" => $subsubcatName,
"amount" => $threerow['value'], "amount" => $threerow['value'],
//"color" => "#".$subCatColor "color" => "#".$subCatColor
); );
$subCatValue += $threerow['value']; $subCatValue += $threerow['value'];
} }
$subnodes[] = Array( $subnodes[] = Array(
"label" => $subcatName, "label" => $subcatName,
"amount" => $subCatValue, "amount" => $subCatValue,
"color" => "#".$subCatColor, "color" => "#".$subCatColor,
"children" => $subsubnodes "children" => $subsubnodes
); );
$catValue += $subCatValue; $catValue += $subCatValue;
} }
$nodes[] = Array( $nodes[] = Array(
"label" => $catName, "label" => $catName,
"amount" => $catValue, "amount" => $catValue,
"color" => "#".$catColor, "color" => "#".$catColor,
"children" => $subnodes "children" => $subnodes
); );
$total += $catValue; $total += $catValue;
} }
$data = Array( $data = Array(
"label" => "Australian Federal Government Contract Spending", "label" => "Australian Federal Government Contract Spending",
"amount" => $total, "amount" => $total,
"color" => "#000000", "color" => "#000000",
"children" => $nodes "children" => $nodes
); );
echo "var data =eval('('+'" . json_encode($data) . "'+')');"; echo "var data =eval('('+'" . json_encode($data) . "'+')');";
?> ?>
   
new BubbleTree({ new BubbleTree({
data: data, data: data,
container: '.bubbletree' container: '.bubbletree'
}); });
}); });
</script> </script>
</head> </head>
<body> <body>
<div class="bubbletree-wrapper"> <div class="bubbletree-wrapper">
<div class="bubbletree"></div> <div class="bubbletree"></div>
</div> </div>
</body> </body>
</html> </html>
   
<?php <?php
   
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
   
  $unspsc = Array();
  $unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';');
  $unspscresult->execute();
  foreach ($unspscresult->fetchAll() as $row) {
  $unspsc[$row['UNSPSC']] = $row['Title'];
  }
   
$query = $conn->prepare(' $query = $conn->prepare('
SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",
  EXTRACT(EPOCH FROM "publishDate") as "publishDate",
  EXTRACT(EPOCH FROM "contractStart") as "contractStart",
  EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",
  value,description,"procurementMethod",category,"categoryUNSPSC",
  (substr( "categoryUNSPSC"::text, 0, 2 ) || \'000000\'::text) as "cat1",
  (substr( "categoryUNSPSC"::text, 0, 4 ) || \'0000\'::text) as "cat2",
  (substr( "categoryUNSPSC"::text, 0, 6 ) || \'00\'::text) as "cat3",
"supplierABN","supplierName", "supplierABN","supplierName",
( (
case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" is null' FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName"
  where "childCN" is null'
, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
// "supplierCity","supplierPostcode","supplierCountry","contactPostcode", // "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3") //
$query->execute(); $query->execute();
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[2] != "") { if ($errors[2] != "") {
die("Export terminated, db error" . print_r($errors, true)); die("Export terminated, db error" . print_r($errors, true));
} }
   
$num_fields = $query->columnCount(); $num_fields = $query->column