Support partial data amendments with multi line descriptions
Support partial data amendments with multi line descriptions

<?php <?php
include_once ("../lib/common.inc.php"); include_once ("../lib/common.inc.php");
function processFile($fpath, $tablename) function processFile($fpath, $tablename)
{ {
global $conn; global $conn;
$row = 1; $row = 1;
$handle = fopen($fpath, "r"); $handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix //"t" mode string translates windows line breaks to unix
$datamapping0711 = array( $datamapping0711 = array(
"Agency" => "agencyName", "Agency" => "agencyName",
"Parent CN ID" => "parentCN", "Parent CN ID" => "parentCN",
"CN ID" => "CNID", "CN ID" => "CNID",
"Publish Date" => "publishDate", "Publish Date" => "publishDate",
"Amendment Date" => "amendDate", "Amendment Date" => "amendDate",
"Status" => "", "Status" => "",
"StartDate" => "contractStart", "StartDate" => "contractStart",
"EndDate" => "contractEnd", "EndDate" => "contractEnd",
"Value" => "value", "Value" => "value",
"Description" => "description", "Description" => "description",
"Agency Ref Id" => "agencyID", "Agency Ref Id" => "agencyID",
"Category" => "category", "Category" => "category",
"Procurement Method" => "procurementMethod", "Procurement Method" => "procurementMethod",
"ATM ID" => "atmID", "ATM ID" => "atmID",
"SON ID" => "SONID", "SON ID" => "SONID",
"Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract" => "confidentialityContract",
"Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
"Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs" => "confidentialityOutputs",
"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
"Consultancy" => "consultancy", "Consultancy" => "consultancy",
"Consultancy Reason(s)" => "consultancyReason", "Consultancy Reason(s)" => "consultancyReason",
"Amendment Reason" => "amendmentReason", "Amendment Reason" => "amendmentReason",
"Supplier Name" => "supplierName", "Supplier Name" => "supplierName",
"Supplier Address" => "supplierAddress", "Supplier Address" => "supplierAddress",
"Supplier City" => "supplierCity", "Supplier City" => "supplierCity",
"Supplier Postcode" => "supplierPostcode", "Supplier Postcode" => "supplierPostcode",
"Supplier Country" => "supplierCountry", "Supplier Country" => "supplierCountry",
"Supplier ABNExempt" => "supplierABNExempt", "Supplier ABNExempt" => "supplierABNExempt",
"Supplier ABN" => "supplierABN", "Supplier ABN" => "supplierABN",
"Agency Branch" => "contactBranch", "Agency Branch" => "contactBranch",
"Agency Divison" => "contactDivision", "Agency Divison" => "contactDivision",
"Agency Postcode" => "contactPostcode", "Agency Postcode" => "contactPostcode",
"" => "" "" => ""
); );
$headers; $headers;
$contractNoticeFields = array( $contractNoticeFields = array(
"importFile", "importFile",
"agencyName", "agencyName",
"parentCN", "parentCN",
"CNID", "CNID",
"publishDate", "publishDate",
"amendDate", "amendDate",
"contractStart", "contractStart",
"contractEnd", "contractEnd",
"value", "value",
"description", "description",
"agencyID", "agencyID",
"category", "category",
"procurementMethod", "procurementMethod",
"atmID", "atmID",
"SONID", "SONID",
"confidentialityContract", "confidentialityContract",
"confidentialityContractReason", "confidentialityContractReason",
"confidentialityOutputs", "confidentialityOutputs",
"confidentialityOutputsReason", "confidentialityOutputsReason",
"consultancy", "consultancy",
"consultancyReason", "consultancyReason",
"amendmentReason", "amendmentReason",
"supplierName", "supplierName",
"supplierAddress", "supplierAddress",
"supplierCity", "supplierCity",
"supplierPostcode", "supplierPostcode",
"supplierCountry", "supplierCountry",
"supplierABNExempt", "supplierABNExempt",
"supplierABN", "supplierABN",
"contactBranch", "contactBranch",
"contactDivision", "contactDivision",
"contactPostcode" "contactPostcode"
); );
$agencyFields = array( $agencyFields = array(
"agencyName" "agencyName"
); );
$supplierFields = array( $supplierFields = array(
"supplierName", "supplierName",
"supplierAddress", "supplierAddress",
"supplierCity", "supplierCity",
"supplierPostcode", "supplierPostcode",
"supplierCountry", "supplierCountry",
"supplierABNExempt", "supplierABNExempt",
"supplierABN" "supplierABN"
); );
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( '; $contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
foreach ($contractNoticeFields as $key => $f) { foreach ($contractNoticeFields as $key => $f) {
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$contractNoticeInsertQ.= ");"; $contractNoticeInsertQ.= ");";
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( '; $supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
foreach ($supplierFields as $key => $f) { foreach ($supplierFields as $key => $f) {
$supplierInsertQ.= ($key == 0 ? "" : ", ") . "?"; $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$supplierInsertQ.= ");"; $supplierInsertQ.= ");";
$supplierInsertQ = $conn->prepare($supplierInsertQ); $supplierInsertQ = $conn->prepare($supplierInsertQ);
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
$agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( '; $agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
foreach ($agencyFields as $key => $f) { foreach ($agencyFields as $key => $f) {
$agencyInsertQ.= ($key == 0 ? "" : ", ") . "?"; $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$agencyInsertQ.= ");"; $agencyInsertQ.= ");";
$agencyInsertQ = $conn->prepare($agencyInsertQ); $agencyInsertQ = $conn->prepare($agencyInsertQ);
} }
while (($data = fgetcsv($handle, 1000, "\t")) !== false) { while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data); $num = count($data);
if ($row == 3) { if ($row == 3) {
$headers = $data; $headers = $data;
} }
elseif ($row > 3) { elseif ($row > 3) {
if ($num > count($datamapping0711)) { if ($num > count($datamapping0711)) {
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data)); die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
} }
$contractNoticeInsert = Array(); $contractNoticeInsert = Array();
$supplierInsert = Array(); $supplierInsert = Array();
$agencyInsert = Array(); $agencyInsert = Array();
$contractNoticeInsert[] = $fpath; $contractNoticeInsert[] = $fpath;
$keys = array_keys($datamapping0711); $keys = array_keys($datamapping0711);
for ($c = 0; $c < $num; $c++) { for ($c = 0; $c < $num; $c++) {
$data[$c] = trim($data[$c], "="); $data[$c] = trim($data[$c], "=");
$data[$c] = trim($data[$c], "\""); $data[$c] = trim($data[$c], "\"");
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
if (in_array(($datamapping0711[$headers[$c]]) , $contractNoticeFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $contractNoticeFields)) {
if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") { if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
$data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers $data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers
if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric"); if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric");
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = 0; $contractNoticeInsert[] = 0;
} }
} }
elseif (($datamapping0711[$headers[$c]]) == "supplierABN") { elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = null; $contractNoticeInsert[] = null;
} }
} }
elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") { elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c])); $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
} }
else { else {
if (strstr("\" =", $data[$c] > 0)) { if (strstr("\" =", $data[$c] > 0)) {
die("Invalid Description field" . $contractNoticeInsert); die("Invalid Description field" . $contractNoticeInsert);
} }
$colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c])); $colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));
   
$contractNoticeInsert[] = $colvalue; $contractNoticeInsert[] = $colvalue;
} }
} }
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
if (in_array(($datamapping0711[$headers[$c]]) , $supplierFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $supplierFields)) {
if (($datamapping0711[$headers[$c]]) == "supplierABN") { if (($datamapping0711[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = 0; $contractNoticeInsert[] = 0;
} }
} }
else { else {
$supplierInsert[] = $data[$c]; $supplierInsert[] = $data[$c];
} }
} }
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
if (in_array(($datamapping0711[$headers[$c]]) , $agencyFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $agencyFields)) {
$agencyInsert[] = $data[$c]; $agencyInsert[] = $data[$c];
} }
} }
} }
flush(); flush();
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ->execute($contractNoticeInsert); $contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
foreach ($contractNoticeFields as $key => $cnf) { foreach ($contractNoticeFields as $key => $cnf) {
echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>"; echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
} }
echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n"; echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
} }
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
$supplierInsertQ->execute($supplierInsert); $supplierInsertQ->execute($supplierInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n"; echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
} }
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
$agencyInsertQ->execute($agencyInsert); $agencyInsertQ->execute($agencyInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n"; echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
} }
} }
flush(); flush();
//echo "<hr>\n"; //echo "<hr>\n";
} }
$row++; $row++;
} }
fclose($handle); fclose($handle);
return $success; return $success;
} }
$path = 'data/'; $path = 'data/';
if ($_REQUEST["fname"] == "") { if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>"; echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path); $dhandle = opendir($path);
// define an array to hold the files // define an array to hold the files
$files = array(); $files = array();
if ($dhandle) { if ($dhandle) {
// loop through all of the files // loop through all of the files
while (false !== ($fname = readdir($dhandle))) { while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) { if (($fname != '.') && ($fname != '..')) {
echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>"; echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
} }
} }
} }
} }
else { else {
$success = 0; $success = 0;
$fname = $_REQUEST["fname"]; $fname = $_REQUEST["fname"];
echo " ============== $fname ============== <br>"; echo " ============== $fname ============== <br>";
flush(); flush();
$success+= processFile($path . $fname, "contractnotice"); $success+= processFile($path . $fname, "contractnotice");
$success+= processFile($path . $fname, "agency"); $success+= processFile($path . $fname, "agency");
$success+= processFile($path . $fname, "supplier"); $success+= processFile($path . $fname, "supplier");
echo "<br> $success records successfully created"; echo "<br> $success records successfully created";
flush(); flush();
} }
// run post import data processing // run post import data processing
// cn // cn
//include ("linkAmendments.php"); include ("linkAmendments.php");
include ("updateUNSPSC.php"); include ("updateUNSPSC.php");
// agency // agency
include ("setAgencyStatus.php"); //include ("setAgencyStatus.php");
include ("setAgencyURLABN.php"); //include ("setAgencyURLABN.php");
?> ?>
   
<?php <?php
include_once ("../../lib/common.inc.php"); include_once ("../../lib/common.inc.php");
/* /*
update contractnotice set "supplierABN" = a."supplierABN" update contractnotice set "supplierABN" = a."supplierABN"
from contractnotice as cn inner join (select "supplierABN", from contractnotice as cn inner join (select "supplierABN",
"supplierName" from contractnotice where "supplierABN" "supplierName" from contractnotice where "supplierABN"
IS NOT NULL and "supplierABN" != 0) as a on IS NOT NULL and "supplierABN" != 0) as a on
cn."supplierName" = a."supplierName" where cn."supplierName" = a."supplierName" where
cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN" cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN"
IS NULL or contractnotice."supplierABN" = 0) */ IS NULL or contractnotice."supplierABN" = 0) */
// http://www.lastcraft.com/browser_documentation.php // http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/ // http://code.google.com/p/phpquery/
require('phpQuery-onefile.php'); require('phpQuery-onefile.php');
function getURL($url) { function getURL($url) {
//return file_get_contents($url); //return file_get_contents($url);
$ch = curl_init($url); $ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 45); curl_setopt($ch, CURLOPT_TIMEOUT, 45);
  curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
  curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
$page = curl_exec($ch); $page = curl_exec($ch);
if (curl_errno($ch)) { if (curl_errno($ch)) {
echo "<font color=red> Database temporarily unavailable: "; echo "<font color=red> Database temporarily unavailable: ";
echo curl_errno($ch) . " " . curl_error($ch); echo curl_errno($ch) . " " . curl_error($ch);
echo $url; echo $url;
echo "</font><br>"; echo "</font><br>";
} }
curl_close($ch); curl_close($ch);
return $page; return $page;
} }
   
   
function getTextFromTHNode($Node, $Text = "") { function getTextFromTHNode($Node, $Text = "") {
if ($Node->tagName == null) if ($Node->tagName == null)
return $Text.$Node->textContent; return $Text.$Node->textContent;
if ($Node->tagName != "td") { if ($Node->tagName != "td") {
$Node = $Node->firstChild; $Node = $Node->firstChild;
if ($Node != null) if ($Node != null)
$Text = getTextFromTHNode($Node, $Text); $Text = getTextFromTHNode($Node, $Text);
   
while($Node->nextSibling != null) { while($Node->nextSibling != null) {
$Text = getTextFromTHNode($Node->nextSibling, $Text); $Text = getTextFromTHNode($Node->nextSibling, $Text);
$Node = $Node->nextSibling; $Node = $Node->nextSibling;
} }
} }
return $Text; return $Text;
} }
   
function getTextFromNode($Node, $Text = "") { function getTextFromNode($Node, $Text = "") {
if ($Node->tagName == null) if ($Node->tagName == null)
return $Text.$Node->textContent; return $Text.$Node->textContent;
if ($Node->tagName != "th" && $Node->tagName != "span") { if ($Node->tagName != "th" && $Node->tagName != "span") {
$Node = $Node->firstChild; $Node = $Node->firstChild;
if ($Node != null) if ($Node != null)
$Text = getTextFromNode($Node, $Text); $Text = getTextFromNode($Node, $Text);
   
while($Node->nextSibling != null) { while($Node->nextSibling != null) {
$Text = getTextFromNode($Node->nextSibling, $Text); $Text = getTextFromNode($Node->nextSibling, $Text);
$Node = $Node->nextSibling; $Node = $Node->nextSibling;
} }
} }
return $Text; return $Text;
} }
function dom_to_array($root) function dom_to_array($root)
{ {
$result = array(); $result = array();
   
if ($root->hasAttributes()) if ($root->hasAttributes())
{ {
$attrs = $root->attributes; $attrs = $root->attributes;
   
foreach ($attrs as $i => $attr) foreach ($attrs as $i => $attr)
$result[$attr->name] = $attr->value; $result[$attr->name] = $attr->value;
} }
   
$children = $root->childNodes; $children = $root->childNodes;
if ($root->childNodes) { if ($root->childNodes) {
if ($children->length == 1) if ($children->length == 1)
{ {
$child = $children->item(0); $child = $children->item(0);
   
if ($child->nodeType == XML_TEXT_NODE) if ($child->nodeType == XML_TEXT_NODE)
{ {
$result['_value'] = $child->nodeValue; $result['_value'] = $child->nodeValue;
   
if (count($result) == 1) if (count($result) == 1)
return $result['_value']; return $result['_value'];
else else
return $result; return $result;
} }
} }
   
$group = array(); $group = array();
   
for($i = 0; $i < $children->length; $i++) for($i = 0; $i < $children->length; $i++)
{ {
$child = $children->item($i); $child = $children->item($i);
   
if (!isset($result[$child->nodeName])) if (!isset($result[$child->nodeName]))
$result[$child->nodeName] = dom_to_array($child); $result[$child->nodeName] = dom_to_array($child);
else else
{ {
if (!isset($group[$child->nodeName])) if (!isset($group[$child->nodeName]))
{ {
$tmp = $result[$child->nodeName]; $tmp = $result[$child->nodeName];
$result[$child->nodeName] = array($tmp); $result[$child->nodeName] = array($tmp);
$group[$child->nodeName] = 1; $group[$child->nodeName] = 1;
} }
   
$result[$child->nodeName][] = dom_to_array($child); $result[$child->nodeName][] = dom_to_array($child);
} }
} }
} }
   
return $result; return $result;
} }
   
function importCN($cnid) { function importCN($cnid) {
global $conn; global $conn;
$CN = str_replace("-A", "00",$cnid); $CN = str_replace("-A", "00",$cnid);
// check if already complete // check if already complete
$query = 'Select "parentCN" from contractnotice $query = 'Select "parentCN" from contractnotice
where "CNID" = :CNID'; where "CNID" = :CNID';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":CNID", $CN); $query->bindParam(":CNID", $CN);
$query->execute(); $query->execute();
$r = $query->fetch(PDO::FETCH_ASSOC); $r = $query->fetch(PDO::FETCH_ASSOC);
if ($r['parentCN'] == NULL) { if ($r['parentCN'] == NULL) {
$site = "https://www.tenders.gov.au/"; $site = "https://www.tenders.gov.au/";
$searchResult = phpQuery::newDocument(getURL("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid)); $searchResult = phpQuery::newDocument(getURL("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
//echo "https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid; //echo "https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid;
$url = ""; $url = "";
foreach(pq('a') as $a) { foreach(pq('a') as $a) {
if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) { if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) {
//echo $a->getAttribute("href"); //echo $a->getAttribute("href");
$url = $a->getAttribute("href"); $url = $a->getAttribute("href");
break; break;
} }
} }
$cn = phpQuery::newDocument(getURL($site.$url)); $cn = phpQuery::newDocument(getURL($site.$url));
$datamapping0711 = array( $datamapping0711 = array(
"Agency" => "agencyName", "Agency" => "agencyName",
"Parent CN" => "parentCN", "Parent CN" => "parentCN",
"CN ID" => "CNID", "CN ID" => "CNID",
"Publish Date" => "publishDate", "Publish Date" => "publishDate",
"Amendment Date" => "amendDate", "Amendment Date" => "amendDate",
"Status" => "", "Status" => "",
"StartDate" => "contractStart", "StartDate" => "contractStart",
"EndDate" => "contractEnd", "EndDate" => "contractEnd",
"Contract Value (AUD)" => "value", "Contract Value (AUD)" => "value",
"Description" => "description", "Description" => "description",
"Agency Reference ID" => "agencyID", "Agency Reference ID" => "agencyID",
"Category" => "category", "Category" => "category",
"Procurement Method" => "procurementMethod", "Procurement Method" => "procurementMethod",
"ATM ID" => "atmID", "ATM ID" => "atmID",
"SON ID" => "SONID", "SON ID" => "SONID",
"Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract" => "confidentialityContract",
"Confidentiality Reason(s) - Contract" => "confidentialityContractReason", "Confidentiality Reason(s) - Contract" => "confidentialityContractReason",
"Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs" => "confidentialityOutputs",
"Confidentiality Reason(s) - Outputs" => "confidentialityOutputsReason", "Confidentiality Reason(s) - Outputs" => "confidentialityOutputsReason",
"Consultancy" => "consultancy", "Consultancy" => "consultancy",
"Consultancy Reason(s)" => "consultancyReason", "Consultancy Reason(s)" => "consultancyReason",
"Amendment Reason" => "amendmentReason", "Amendment Reason" => "amendmentReason",
"Name" => "supplierName", "Name" => "supplierName",
"Postal Address" => "supplierAddress", "Postal Address" => "supplierAddress",
"Town/City" => "supplierCity", "Town/City" => "supplierCity",
"Postcode" => "supplierPostcode", "Postcode" => "supplierPostcode",
"Country" => "supplierCountry", "Country" => "supplierCountry",
"ABN Exempt" => "supplierABNExempt", "ABN Exempt" => "supplierABNExempt",
"ABN" => "supplierABN", "ABN" => "supplierABN",
"Branch" => "contactBranch", "Branch" => "contactBranch",
"Division" => "contactDivision", "Division" => "contactDivision",
"Office Postcode" => "contactPostcode" "Office Postcode" => "contactPostcode"
); );
$cnFields = Array(); $cnFields = Array();
foreach(pq('tr') as $tr) { foreach(pq('tr') as $tr) {
$tra = dom_to_array($tr); $tra = dom_to_array($tr);
if (is_array($tra['th'])) { if (is_array($tra['th'])) {
$fieldName = trim(getTextFromTHNode($tr)); $fieldName = trim(getTextFromTHNode($tr));
} else { } else {
$fieldName = trim(str_replace("/th>","",$tra['th'])); $fieldName = trim(str_replace("/th>","",$tra['th']));
} }
$fieldValue = trim(print_r($tra['td'],true)); $fieldValue = trim(print_r($tra['td'],true));
if ($fieldName == "State/Territory" || $fieldName == "Contact Name" if ($fieldName == "State/Territory" || $fieldName == "Contact Name"
|| $fieldName == "Contact Phone" || $fieldName == "Contact Email" || $fieldName == "Contact Phone" || $fieldName == "Contact Email"
||$fieldName == "Amendments") { ||$fieldName == "Amendments") {
// do nothing // do nothing
} else if ($fieldName == "Contract Period") { } else if ($fieldName == "Contract Period") {
$contractPeriod = explode("to",$fieldValue); $contractPeriod = explode("to",$fieldValue);
$cnFields["contractStart"] = trim($contractPeriod[0]); $cnFields["contractStart"] = trim($contractPeriod[0]);
$cnFields["contractEnd"] = trim($contractPeriod[1]); $cnFields["contractEnd"] = trim($contractPeriod[1]);
} else { } else {
$fieldName = $datamapping0711[$fieldName]; $fieldName = $datamapping0711[$fieldName];
   
if ($fieldName == "parentCN" || $fieldName == "CNID") { if ($fieldName == "parentCN" || $fieldName == "CNID") {
if (is_array($tra['td'])) { if (is_array($tra['td'])) {
$fieldValue = trim(getTextFromNode($tr)); $fieldValue = trim(getTextFromNode($tr));
} }
$fieldValue = substr($fieldValue, 2); // take off the "CN" prefix $fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
$fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers $fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
} elseif ($fieldName == "description") { } elseif ($fieldName == "description") {
if (is_array($tra['td'])) $fieldValue = print_r($tra['td']['p'],true); if (is_array($tra['td'])) $fieldValue = print_r($tra['td']['p'],true);
} elseif ($fieldName == "value" || $fieldName == "supplierABN") { } elseif ($fieldName == "value" || $fieldName == "supplierABN") {
if (is_array($tra['td'])) { if (is_array($tra['td'])) {
$fieldValue = trim(getTextFromNode($tr)); $fieldValue = trim(getTextFromNode($tr));
} }
$fieldValue = str_replace(Array("$",","," "), "", $fieldValue); $fieldValue = str_replace(Array("$",","," "), "", $fieldValue);
//if (!is_numeric($fieldValue)) $fieldValue = 0; //if (!is_numeric($fieldValue)) $fieldValue = 0;
if ($fieldValue == "Exempt") $fieldValue = NULL; if ($fieldValue == "Exempt") $fieldValue = NULL;
} elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") { } elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
$fieldValue = date('Y-m-d H:i:s', strtotime($fieldValue)); $fieldValue = date('Y-m-d H:i:s', strtotime($fieldValue));
} elseif (is_array($tra['td'])) { } elseif (is_array($tra['td'])) {
$fieldValue = trim(getTextFromNode($tr)); $fieldValue = trim(getTextFromNode($tr));
} }
echo $fieldName. " = " .$fieldValue."<br>\n"; echo $fieldName. " = " .$fieldValue."<br>\n";
$cnFields[$fieldName] = $fieldValue; $cnFields[$fieldName] = $fieldValue;
} }
} }
  if (isset($cnFields[""])) {
  $cnFields["description"] .= $cnFields[""];
  unset($cnFields[""]);
  }
$cnFields["importFile"] = $url; $cnFields["importFile"] = $url;
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( '; $contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
for($key = 0; $key < sizeof($cnFields); $key++) { for($key = 0; $key < sizeof($cnFields); $key++) {
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$contractNoticeInsertQ.= ");"; $contractNoticeInsertQ.= ");";
//echo $contractNoticeInsertQ; //echo $contractNoticeInsertQ;
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
$contractNoticeInsertQ->execute(array_values($cnFields)); $contractNoticeInsertQ->execute(array_values($cnFields));
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
echo "Dupe {$cnFields['CNID']}<br>"; echo "Dupe {$cnFields['CNID']}<br>";
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
echo "Success insert {$cnFields['CNID']} <br>"; echo "Success insert {$cnFields['CNID']} <br>";
} }
else { else {
foreach ($cnFields as $key => $cnf) { foreach ($cnFields as $key => $cnf) {
echo var_dump($key) . $cnf . "<br>"; echo var_dump($key) . $cnf . "<br>";
} }
echo $cnFields['CNID'] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n"; echo $cnFields['CNID'] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
} }
$contractNoticeUpdateQ = 'UPDATE contractnotice SET '; $contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
$count = 0; $count = 0;
foreach ($cnFields as $key => $f) { foreach ($cnFields as $key => $f) {
$count++; $count++;
$contractNoticeUpdateQ.= '"'.$key.'"=? '.($count >= sizeof($cnFields) ? "" : ", "); $contractNoticeUpdateQ.= '"'.$key.'"=? '.($count >= sizeof($cnFields) ? "" : ", ");
} }
$contractNoticeUpdateQ.= ' WHERE "CNID"=?;'; $contractNoticeUpdateQ.= ' WHERE "CNID"=?;';
$cnFields[] = $cnFields["CNID"]; $cnFields[] = $cnFields["CNID"];
//echo $contractNoticeUpdateQ; //echo $contractNoticeUpdateQ;
$contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ); $contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
$contractNoticeUpdateQ->execute(array_values($cnFields)); $contractNoticeUpdateQ->execute(array_values($cnFields));
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
print_r($errors); print_r($errors);
echo "Dupe update {$cnFields['CNID']}<br>"; echo "Dupe update {$cnFields['CNID']}<br>";
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
echo "Success update {$cnFields['CNID']} <br>"; echo "Success update {$cnFields['CNID']} <br>";
} }
else { else {
foreach ($cnFields as $key => $cnf) { foreach ($cnFields as $key => $cnf) {
echo var_dump($key) . $cnf . "<br>"; echo var_dump($key) . $cnf . "<br>";
} }
echo $cnFields['CNID'] . " failed CN update.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n"; echo $cnFields['CNID'] . " failed CN update.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
} }
} }
} }
function processFile($fpath, $tablename) function processFile($fpath, $tablename)
{ {
global $conn; global $conn;
echo " ============== $fpath ============== <br>"; echo " ============== $fpath ============== <br>";
$handle = fopen($fpath, "r"); $handle = fopen($fpath, "r");
flush(); flush();
$row = 1; $row = 1;
while (($data = fgetcsv($handle, 1000, "\t")) !== false) { while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
if ($row > 3) { if ($row > 3) {
$data[0] = trim($data[0], "="); $data[0] = trim($data[0], "=");
$data[0] = trim($data[0], "\""); $data[0] = trim($data[0], "\"");
if (strpos($data[0], "-A") > 0) { if (strpos($data[0], "-A") > 0) {
echo "Loading {$data[0]} ... <br>\n"; echo "Loading {$data[0]} ... <br>\n";
importCN(str_replace("CN","",$data[0])); importCN(str_replace("CN","",$data[0]));
} }
} }
flush(); flush();
//echo "<hr>\n"; //echo "<hr>\n";
$row++; $row++;
} }
fclose($handle); fclose($handle);
} }
$path = './'; $path = './';
if ($_REQUEST["fname"] == "") { if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>"; echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path); $dhandle = opendir($path);
// define an array to hold the files // define an array to hold the files
$files = array(); $files = array();
if ($dhandle) { if ($dhandle) {
// loop through all of the files // loop through all of the files
while (false !== ($fname = readdir($dhandle))) { while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) { if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) {
echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>"; echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice"); processFile($path . $fname, "contractnotice");
} }
} }
} }
} }
else { else {
$success = 0; $success = 0;
$fname = $_REQUEST["fname"]; $fname = $_REQUEST["fname"];
$success+= processFile($path . $fname, "contractnotice"); $success+= processFile($path . $fname, "contractnotice");
   
} }
   
?> ?>
   
<?php <?php
$includedFlot = false; $includedFlot = false;
function includeFlot() { function includeFlot() {
if (!$includedFlot) { if (!$includedFlot) {
echo ' <!--[if lte IE 8]><script language="javascript" type="text/javascript" src="lib/flot/excanvas.min.js"></script><![endif]--> echo ' <!--[if lte IE 8]><script language="javascript" type="text/javascript" src="lib/flot/excanvas.min.js"></script><![endif]-->
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script>
  <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.stack.js"></script>
   
'; ';
$includedFlot = true; $includedFlot = true;
} }
} }
   
function CNDistributionGraph() { function CNDistributionGraph() {
global $conn; global $conn;
includeFlot(); includeFlot();
?> ?>
<center><div id="cndist" style="width:900px;height:550px"></div></center> <center><div id="cndist" style="width:900px;height:550px"></div></center>
<script type="text/javascript"> <script type="text/javascript">
var placeholder = $("#cndist"); var placeholder = $("#cndist");
$(function () { $(function () {
   
var d1 = []; var d1 = [];
  var d2 = [];
<?php <?php
$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999) as a group by cnid order by cnid'; $query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is null) as a group by cnid order by cnid';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
   
foreach ($query->fetchAll() as $delta) { foreach ($query->fetchAll() as $delta) {
   
echo "d1.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n"; echo "d1.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
  };
  $query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid';
  $query = $conn->prepare($query);
  $query->execute();
  if (!$query) {
  databaseError($conn->errorInfo());
  return Array();
  }
   
  foreach ($query->fetchAll() as $delta) {
   
  echo "d2.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
}; };
?> ?>
   
var data = [ var data = [
{ {
data: d1, data: d1,
series: { series: {
lines: { show: true }, lines: { show: true },
points: { show: true } points: { show: true }
}, },
bars: { show: true } bars: { show: true }
}, },
  {
  data: d2,
  series: {
  lines: { show: true },
  points: { show: true }
  },
  bars: { show: true }
  },
]; ];
var options = var options =
{ {
  series: {
  stack: true,
  },
grid: { hoverable: true, clickable: true, labelMargin: 17 }, grid: { hoverable: true, clickable: true, labelMargin: 17 },
selection: { mode: "x" } selection: { mode: "x" }
}; };
   
placeholder.bind("plotselected", function (event, ranges) { placeholder.bind("plotselected", function (event, ranges) {
plot = $.plot(placeholder, data, plot = $.plot(placeholder, data,
$.extend(true, {}, options, { $.extend(true, {}, options, {
xaxis: { min: ranges.xaxis.from, max: ranges.xaxis.to } xaxis: { min: ranges.xaxis.from, max: ranges.xaxis.to }
})); }));
}); });
var previousPoint = null; var previousPoint = null;
placeholder.bind("plothover", function (event, pos, item) { placeholder.bind("plothover", function (event, pos, item) {
$("#x").text(pos.x.toFixed(2)); $("#x").text(pos.x.toFixed(2));
$("#y").text(pos.y.toFixed(2)); $("#y").text(pos.y.toFixed(2));
if (item) { if (item) {
if (previousPoint != item.dataIndex) { if (previousPoint != item.dataIndex) {
previousPoint = item.dataIndex; previousPoint = item.dataIndex;
$("#tooltip").remove(); $("#tooltip").remove();
var x = item.datapoint[0].toFixed(2), var x = item.datapoint[0].toFixed(2),
y = item.datapoint[1].toFixed(2); y = item.datapoint[1].toFixed(2);
showTooltip(item.pageX, item.pageY, showTooltip(item.pageX, item.pageY,
item.series.label + " of " + x + " = " + y); item.series.label + " of " + x + " = " + y);
} }
} }
else { else {
$("#tooltip").remove(); $("#tooltip").remove();
previousPoint = null; previousPoint = null;
} }
}); });
   
var plot = $.plot(placeholder, data, var plot = $.plot(placeholder, data,
options); options);
}); });
   
function showTooltip(x, y, contents) { function showTooltip(x, y, contents) {
$('<div id="tooltip">' + contents + '</div>').css( { $('<div id="tooltip">' + contents + '</div>').css( {
position: 'absolute', position: 'absolute',
display: 'none', display: 'none',
top: y + 5, top: y + 5,
left: x + 5, left: x + 5,
border: '1px solid #fdd', border: '1px solid #fdd',
padding: '2px', padding: '2px',
'background-color': '#fee', 'background-color': '#fee',
opacity: 0.80 opacity: 0.80
}).appendTo("body").fadeIn(200); }).appendTo("body").fadeIn(200);
} }
</script> </script>
<?php <?php
} }
   
function agenciesGraph() { function agenciesGraph() {
global $conn; global $conn;
includeFlot(); includeFlot();
$query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" WHERE "childCN" = 0 $query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" WHERE "childCN" = 0
GROUP BY abn ORDER BY SUM("value") DESC'; GROUP BY abn ORDER BY SUM("value") DESC';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
?> ?>
<script type="text/javascript"> <script type="text/javascript">
$(function () { $(function () {
// data // data
var data = [ var data = [
<?php <?php
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
echo '{ label: "'.$row['agencyname'].'", data: '.doubleval($row["val"]).'},'; echo '{ label: "'.$row['agencyname'].'", data: '.doubleval($row["val"]).'},';
} }
?> ?>
]; ];
// GRAPH 7 // GRAPH 7
$.plot($("#graph7"), data, $.plot($("#graph7"), data,
{ {
series: { series: {
pie: { pie: {
show: true, show: true,
radius: 1, radius: 1,
tilt: 0.75, tilt: 0.75,
label: { label: {
show: true, show: true,
radius: 1, radius: 1,
formatter: function(label, series){ formatter: function(label, series){
return '<div style="font-size:8pt;text-align:center;padding:2px;color:white;">'+label+': '+Math.round(series.percent)+'%</div>'; return '<div style="font-size:8pt;text-align:center;padding:2px;color:white;">'+label+': '+Math.round(series.percent)+'%</div>';
}, },
background: { background: {
opacity: 0.5, opacity: 0.5,
color: '#000' color: '#000'
} }
}, },
combine: { combine: {
color: '#999', color: '#999',
threshold: 0.012 threshold: 0.012
} }
} }
}, },
legend: { legend: {
show: false show: false
} }
}); });
}); });
</script> </script>
<div id="graph7" style="width:900px;height:550px"></div> <div id="graph7" style="width:900px;height:550px"></div>
   
<?php <?php
}; };
   
function agencySuppliersGraph($agency) { function agencySuppliersGraph($agency) {
$agency = "AusAid"; $agency = "AusAid";
$topX = 15; $topX = 15;
$query = "SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND childCN = 0 AND agencyName = '$agency' $query = "SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND childCN = 0 AND agencyName = '$agency'
GROUP BY lower(supplierName) ORDER BY val DESC limit $topX"; GROUP BY lower(supplierName) ORDER BY val DESC limit $topX";
$result = mysql_query($query); $result = mysql_query($query);
$suppliers = Array(); $suppliers = Array();
$values = Array(); $values = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
$suppliers[] = ucsmart($row['supplierName']); $suppliers[] = ucsmart($row['supplierName']);
$values[] = doubleval($row["val"]); $values[] = doubleval($row["val"]);
} }
mysql_free_result($result); mysql_free_result($result);
   
$query = "SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND childCN = 0 and agencyName = '$agency' $query = "SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND childCN = 0 and agencyName = '$agency'
GROUP BY lower(supplierName) ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a"; GROUP BY lower(supplierName) ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a";
$result = mysql_query($query); $result = mysql_query($query);
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
if ($row['count'] > 0) { if ($row['count'] > 0) {
$suppliers[] = $row['count'] . " other suppliers"; $suppliers[] = $row['count'] . " other suppliers";
$values[] = doubleval($row[0]); $values[] = doubleval($row[0]);
} }
} }
mysql_free_result($result); mysql_free_result($result);
} }
   
function CnCGraph() { function CnCGraph() {
$query = "select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` $query = "select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month"; where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month";
$result = mysql_query($query); $result = mysql_query($query);
$methods = Array("Direct","Open","Select"); $methods = Array("Direct","Open","Select");
$dates = Array(); $dates = Array();
$methodCountsP = Array(); $methodCountsP = Array();
$methodCounts = Array(); $methodCounts = Array();
$maxValue = 0; $maxValue = 0;
while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) { while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
if ($row['val'] > $maxValue) $maxValue = $row['val']; if ($row['val'] > $maxValue) $maxValue = $row['val'];
$date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"]; $date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"];
if (array_search($date,$dates) === false ) { if (array_search($date,$dates) === false ) {
$dates[$row["year"]*100 + $row["month"]] = $date; $dates[$row["year"]*100 + $row["month"]] = $date;
ksort($dates); ksort($dates);
} }
$methodCountsP[$row["procurementMethod"]][$date] = $row["count"]; $methodCountsP[$row["procurementMethod"]][$date] = $row["count"];
   
} }
foreach ($methods as $method) { foreach ($methods as $method) {
foreach($dates as $date) { foreach($dates as $date) {
if ($methodCountsP[$method][$date] > 0) $methodCounts[$method][] = $methodCountsP[$method][$date]; if ($methodCountsP[$method][$date] > 0) $methodCounts[$method][] = $methodCountsP[$method][$date];
else $methodCounts[$method][] = 0; else $methodCounts[$method][] = 0;
} }
} }
$dates = array_values($dates); $dates = array_values($dates);
$totalRecords = array_sum_all($methodCounts); $totalRecords = array_sum_all($methodCounts);
mysql_free_result($result); mysql_free_result($result);
   
   
function formatCallback($aVal) { function formatCallback($aVal) {
global $totalRecords; global $totalRecords;
return percent($aVal, $totalRecords) . "%"; return percent($aVal, $totalRecords) . "%";
} }
$attributes = Array(); $attributes = Array();
$attributeNames = Array( $attributeNames = Array(
"Consultancies", "Consultancies",
"Confidentialities" "Confidentialities"
); );
$query = "SELECT 'consultancy', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ consultancy='Yes' AND childCN = 0;"; $query = "SELECT 'consultancy', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ consultancy='Yes' AND childCN = 0;";
$result = mysql_query($query); $result = mysql_query($query);
$row = mysql_fetch_array($result, MYSQL_BOTH); $row = mysql_fetch_array($result, MYSQL_BOTH);
$attributes[0] = $row[1]; $attributes[0] = $row[1];
$query = "SELECT 'confidentiality', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ (confidentialityContract='Yes' OR confidentialityOutputs='Yes') AND childCN = 0;"; $query = "SELECT 'confidentiality', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ (confidentialityContract='Yes' OR confidentialityOutputs='Yes') AND childCN = 0;";
$result = mysql_query($query); $result = mysql_query($query);
$row = mysql_fetch_array($result, MYSQL_BOTH); $row = mysql_fetch_array($result, MYSQL_BOTH);
$attributes[1] = $row[1]; $attributes[1] = $row[1];
mysql_free_result($result); mysql_free_result($result);
} }
function ContractPublishedGraph() { function ContractPublishedGraph() {
$query = "SELECT YEAR(publishDate), MONTH(publishDate), $query = "SELECT YEAR(publishDate), MONTH(publishDate),
SUM(value) as val, count(1) as count FROM `contractnotice` SUM(value) as val, count(1) as count FROM `contractnotice`
WHERE (YEAR(publishDate) >= 2008) WHERE (YEAR(publishDate) >= 2008)
AND childCN = 0 AND childCN = 0
GROUP BY MONTH(publishDate), YEAR(publishDate) GROUP BY MONTH(publishDate), YEAR(publishDate)
ORDER BY YEAR(publishDate), MONTH(publishDate)"; ORDER BY YEAR(publishDate), MONTH(publishDate)";
   
$result = mysql_query($query); $result = mysql_query($query);
$dates = Array(); $dates = Array();
$values = Array(); $values = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row["val"]),2); $value = number_format(doubleval($row["val"]),2);
$month_name = date( 'F', mktime(0, 0, 0, $row[1]) ); $month_name = date( 'F', mktime(0, 0, 0, $row[1]) );
$dates[] = $month_name." {$row[0]}"; $dates[] = $month_name." {$row[0]}";
$counts[] = doubleval($row["count"]); $counts[] = doubleval($row["count"]);
$values[] = doubleval($row["val"]); $values[] = doubleval($row["val"]);
} }
mysql_free_result($result); mysql_free_result($result);
   
} }
function ContractStartingGraph() { function ContractStartingGraph() {
$query = "SELECT YEAR(contractStart), MONTH(contractStart), $query = "SELECT YEAR(contractStart), MONTH(contractStart),
SUM(value) as val, count(1) as count FROM `contractnotice` SUM(value) as val, count(1) as count FROM `contractnotice`
WHERE (YEAR(contractStart) >= 2008) WHERE (YEAR(contractStart) >= 2008)
AND childCN = 0 AND childCN = 0
GROUP BY MONTH(contractStart), YEAR(contractStart) GROUP BY MONTH(contractStart), YEAR(contractStart)
ORDER BY YEAR(contractStart), MONTH(contractStart)"; ORDER BY YEAR(contractStart), MONTH(contractStart)";
   
$result = mysql_query($query); $result = mysql_query($query);
$dates = Array(); $dates = Array();
$values = Array(); $values = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row["val"]),2); $value = number_format(doubleval($row["val"]),2);
$month_name = date( 'F', mktime(0, 0, 0, $row[1]) ); $month_name = date( 'F', mktime(0, 0, 0, $row[1]) );
$dates[] = $month_name." {$row[0]}"; $dates[] = $month_name." {$row[0]}";
$counts[] = doubleval($row["count"]); $counts[] = doubleval($row["count"]);
$values[] = doubleval($row["val"]); $values[] = doubleval($row["val"]);
} }
mysql_free_result($result); mysql_free_result($result);
} }
function MethodCountGraph() { function MethodCountGraph() {
$query = "select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` $query = "select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month"; where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month";
$result = mysql_query($query); $result = mysql_query($query);
$methods = Array("Direct","Open","Select"); $methods = Array("Direct","Open","Select");
$dates = Array(); $dates = Array();
$methodCountsP = Array(); $methodCountsP = Array();
$methodCounts = Array(); $methodCounts = Array();
$maxValue = 0; $maxValue = 0;
while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) { while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
if ($row['value'] > $maxValue) $maxValue = $row['value']; if ($row['value'] > $maxValue) $maxValue = $row['value'];
$date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"]; $date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"];
if (array_search($date,$dates) === false ) { if (array_search($date,$dates) === false ) {
$dates[$row["year"]*100 + $row["month"]] = $date; $dates[$row["year"]*100 + $row["month"]] = $date;
ksort($dates); ksort($dates);
} }
$methodCountsP[$row["procurementMethod"]][$date] = $row["count"]; $methodCountsP[$row["procurementMethod"]][$date] = $row["count"];
   
} }
foreach ($methods as $method) { foreach ($methods as $method) {
foreach($dates as $date) { foreach($dates as $date) {
if ($methodCountsP[$method][$date] > 0) $methodCounts[$method][] = $methodCountsP[$method][$date]; if ($methodCountsP[$method][$date] > 0) $methodCounts[$method][] = $methodCountsP[$method][$date];
else $methodCounts[$method][] = 0; else $methodCounts[$method][] = 0;
} }
} }
$dates = array_values($dates); $dates = array_values($dates);
$totalRecords = array_sum_all($methodCounts); $totalRecords = array_sum_all($methodCounts);
mysql_free_result($result); mysql_free_result($result);
} }
function MethodValueGraph() { function MethodValueGraph() {
$query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` $query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month"; where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month";
$result = mysql_query($query); $result = mysql_query($query);
$methods = Array("Direct","Open","Select"); $methods = Array("Direct","Open","Select");
$dates = Array(); $dates = Array();
$methodValuesP = Array(); $methodValuesP = Array();
$methodValues = Array(); $methodValues = Array();
$maxValue = 0; $maxValue = 0;
while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) { while ($row = mysql_fetch_array($result, MYSQL_ASSOC)) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
if ($row['value'] > $maxValue) $maxValue = $row['value']; if ($row['value'] > $maxValue) $maxValue = $row['value'];
$date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"]; $date = date( 'F ', mktime(0, 0, 0, $row["month"]) ). $row["year"];
if (array_search($date,$dates) === false ) { if (array_search($date,$dates) === false ) {
$dates[$row["year"]*100 + $row["month"]] = $date; $dates[$row["year"]*100 + $row["month"]] = $date;
ksort($dates); ksort($dates);
} }
$methodValuesP[$row["procurementMethod"]][$date] = $row["value"]; $methodValuesP[$row["procurementMethod"]][$date] = $row["value"];
   
} }
foreach ($methods as $method) { foreach ($methods as $method) {
foreach($dates as $date) { foreach($dates as $date) {
if ($methodValuesP[$method][$date] > 0) $methodValues[$method][] = $methodValuesP[$method][$date]; if ($methodValuesP[$method][$date] > 0) $methodValues[$method][] = $methodValuesP[$method][$date];
else $methodValues[$method][] = 0; else $methodValues[$method][] = 0;
} }
} }
$dates = array_values($dates); $dates = array_values($dates);
$totalRecords = array_sum_all($methodValues); $totalRecords = array_sum_all($methodValues);
mysql_free_result($result); mysql_free_result($result);
   
} }
function SuppliersGraph() { function SuppliersGraph() {
   
global $conn; global $conn;
includeFlot(); includeFlot();
$topX = 10; $topX = 10;
$suppliers = Array(); $suppliers = Array();
$values = Array(); $values = Array();
   
   
$query = 'SELECT SUM("value") as value, MAX("supplierName") as supplierName, ( $query = 'SELECT SUM("value") as value, MAX("supplierName") as supplierName, (
case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID
FROM contractnotice FROM contractnotice
WHERE "childCN" = 0 WHERE "childCN" = 0
GROUP BY supplierID GROUP BY supplierID
ORDER BY value DESC ORDER BY value DESC
LIMIT '.$topX; LIMIT '.$topX;
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} }
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
// $value = number_format(doubleval($row["value"]) , 2); // $value = number_format(doubleval($row["value"]) , 2);
$suppliers[] = ucsmart($row[1]); $suppliers[] = ucsmart($row[1]);
$values[] = doubleval($row["value"]); $values[] = doubleval($row["value"]);
} }
   
$query = 'SELECT sum(a.svalue) as val, suppliercountry from (SELECT sum("value") as svalue, max("supplierCountry") as suppliercountry $query = 'SELECT sum(a.svalue) as val, suppliercountry from (SELECT sum("value") as svalue, max("supplierCountry") as suppliercountry
FROM contractnotice WHERE "childCN" = 0 and "supplierCountry" NOT ILIKE \'Australia\' FROM contractnotice WHERE "childCN" = 0 and "supplierCountry" NOT ILIKE \'Australia\'
GROUP BY "supplierName" ORDER BY svalue LIMIT 18446744073 OFFSET 10) as a group by suppliercountry order by val DESC limit 10 '; GROUP BY "supplierName" ORDER BY svalue LIMIT 18446744073 OFFSET 10) as a group by suppliercountry order by val DESC limit 10 ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} }
   
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$suppliers[] = "Other suppliers in ".ucsmart($row["suppliercountry"]); $suppliers[] = "Other suppliers in ".ucsmart($row["suppliercountry"]);
$values[] = doubleval($row[0]); $values[] = doubleval($row[0]);
} }
   
   
$query = 'SELECT sum(a.value) as val, substring( $query = 'SELECT sum(a.value) as val, substring(
supplierpostcode from 0 for 2) as postcode from (SELECT sum(value) as value, max("supplierPostcode") as supplierpostcode, max("supplierCountry") as suppliercountry supplierpostcode from 0 for 2) as postcode from (SELECT sum(value) as value, max("supplierPostcode") as supplierpostcode, max("supplierCountry") as suppliercountry
FROM contractnotice WHERE "childCN" = 0 FROM contractnotice WHERE "childCN" = 0
GROUP BY "supplierABN" ORDER BY sum(value) LIMIT 1844674 OFFSET 10) as a GROUP BY "supplierABN" ORDER BY sum(value) LIMIT 1844674 OFFSET 10) as a
WHERE (suppliercountry ILIKE \'Australia\') WHERE (suppliercountry ILIKE \'Australia\')
group by substring( group by substring(
supplierpostcode from 0 for 2) supplierpostcode from 0 for 2)
order by val DESC;'; order by val DESC;';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} }
   
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
if ($row['postcode'][0] == 2 && $row['postcode'][1] == 6) $ACTvalue += $row[0]; if ($row['postcode'][0] == 2 && $row['postcode'][1] == 6) $ACTvalue += $row[0];
else if ($row['postcode'][0] == 2 || $row['postcode'][0] == 1) $NSWvalue += $row[0]; else if ($row['postcode'][0] == 2 || $row['postcode'][0] == 1) $NSWvalue += $row[0];
else if ($row['postcode'][0] == 3 || $row['postcode'][0] == 8) $Vicvalue += $row[0]; else if ($row['postcode'][0] == 3 || $row['postcode'][0] == 8) $Vicvalue += $row[0];
else if ($row['postcode'][0] == 4 || $row['postcode'][0] == 9) $QLDvalue += $row[0]; else if ($row['postcode'][0] == 4 || $row['postcode'][0] == 9) $QLDvalue += $row[0];
else if ($row['postcode'][0] == 5) $SAvalue += $row[0]; else if ($row['postcode'][0] == 5) $SAvalue += $row[0];
else if ($row['postcode'][0] == 6) $WAvalue += $row[0]; else if ($row['postcode'][0] == 6) $WAvalue += $row[0];
else if ($row['postcode'][0] == 7) $Tasvalue += $row[0]; else if ($row['postcode'][0] == 7) $Tasvalue += $row[0];
else if ($row['postcode'][0] == 0) $NTvalue += $row[0]; else if ($row['postcode'][0] == 0) $NTvalue += $row[0];
} }
$suppliers[] = "Other suppliers in Australia - ACT"; $suppliers[] = "Other suppliers in Australia - ACT";
$values[] = doubleval($ACTvalue); $values[] = doubleval($ACTvalue);
$suppliers[] = "Other suppliers in Australia - NSW"; $suppliers[] = "Other suppliers in Australia - NSW";
$values[] = doubleval($NSWvalue); $values[] = doubleval($NSWvalue);
$suppliers[] = "Other suppliers in Australia - Victoria"; $suppliers[] = "Other suppliers in Australia - Victoria";
$values[] = doubleval($Vicvalue); $values[] = doubleval($Vicvalue);
$suppliers[] = "Other suppliers in Australia - Queensland"; $suppliers[] = "Other suppliers in Australia - Queensland";
$values[] = doubleval($QLDvalue); $values[] = doubleval($QLDvalue);
$suppliers[] = "Other suppliers in Australia - NT"; $suppliers[] = "Other suppliers in Australia - NT";
$values[] = doubleval($NTvalue); $values[] = doubleval($NTvalue);
$suppliers[] = "Other suppliers in Australia - West Australia"; $suppliers[] = "Other suppliers in Australia - West Australia";
$values[] = doubleval($WAvalue); $values[] = doubleval($WAvalue);
$suppliers[] = "Other suppliers in Australia - South Australia"; $suppliers[] = "Other suppliers in Australia - South Australia";
$values[] = doubleval($SAvalue); $values[] = doubleval($SAvalue);
$suppliers[] = "Other suppliers in Australia - Tasmania"; $suppliers[] = "Other suppliers in Australia - Tasmania";
$values[] = doubleval($Tasvalue); $values[] = doubleval($Tasvalue);
   
   
?> ?>
<script type="text/javascript"> <script type="text/javascript">
$(function () { $(function () {
// data // data
var data = [ var data = [
<?php <?php
foreach ($suppliers as $key => $supplier) { foreach ($suppliers as $key => $supplier) {
echo '{ label: "'.$supplier.'", data: '.doubleval($values[$key]).'},'; echo '{ label: "'.$supplier.'", data: '.doubleval($values[$key]).'},';
} }
?> ?>
]; ];
// GRAPH 7 // GRAPH 7
$.plot($("#graph7"), data, $.plot($("#graph7"), data,
{ {
series: { series: {
pie: { pie: {
show: true, show: true,
radius: 1, radius: 1,
tilt: 0.75, tilt: 0.75,
label: { label: {
show: true, show: true,
radius: 1, radius: 1,
formatter: function(label, series){ formatter: function(label, series){
return '<div style="font-size:8pt;text-align:center;padding:2px;color:white;">'+label+': '+Math.round(series.percent)+'%</div>'; return '<div style="font-size:8pt;text-align:center;padding:2px;color:white;">'+label+': '+Math.round(series.percent)+'%</div>';
}, },
background: { background: {
opacity: 0.5, opacity: 0.5,
color: '#000' color: '#000'
} }
}, },
combine: { combine: {
color: '#999', color: '#999',
threshold: 0.012 threshold: 0.012
} }
} }
}, },
legend: { legend: {
show: false show: false
} }
}); });
}); });
</script> </script>
<div id="graph7" style="width:900px;height:550px"></div> <div id="graph7" style="width:900px;height:550px"></div>
   
<?php <?php
} }
   
?> ?>