gitphp 0.2.9.1 :: contractdashboard.git/commitdiff

add datagov importer

6 files changed: (show all)
admin/import.php
admin/importdatagov.php (new)
admin/partialdata/import.php
admin/partialdata/importamendments.php
heuristics/historyHeuristics.php
lib/common.inc.php

file:a/admin/import.php -> file:b/admin/import.php

<?php	<?php
if (php_sapi_name() != "cli") {	if (php_sapi_name() != "cli") {

include_once ("../lib/common.inc.php");	include_once ("../lib/common.inc.php");
auth();	auth();
$contractNoticeFields = array(	$contractNoticeFields = array(
"importFile",	"importFile",
"agencyName",	"agencyName",
"parentCN",	"parentCN",
"CNID",	"CNID",
"publishDate",	"publishDate",
"amendDate",	"amendDate",
"contractStart",	"contractStart",
"contractEnd",	"contractEnd",
"value",	"value",
"description",	"description",
"agencyID",	"agencyID",
"category",	"category",
"procurementMethod",	"procurementMethod",
"atmID",	"atmID",
"SONID",	"SONID",
"confidentialityContract",	"confidentialityContract",
"confidentialityContractReason",	"confidentialityContractReason",
"confidentialityOutputs",	"confidentialityOutputs",
"confidentialityOutputsReason",	"confidentialityOutputsReason",
"consultancy",	"consultancy",
"consultancyReason",	"consultancyReason",
"amendmentReason",	"amendmentReason",
"supplierName",	"supplierName",
"supplierAddress",	"supplierAddress",
"supplierCity",	"supplierCity",
"supplierPostcode",	"supplierPostcode",
"supplierCountry",	"supplierCountry",
"supplierABNExempt",	"supplierABNExempt",
"supplierABN",	"supplierABN",
"contactBranch",	"contactBranch",
"contactDivision",	"contactDivision",
"contactPostcode"	"contactPostcode"
);	);

$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';	$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
foreach ($contractNoticeFields as $key => $f) {	foreach ($contractNoticeFields as $key => $f) {
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";	$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
}	}
$contractNoticeInsertQ.= ");";	$contractNoticeInsertQ.= ");";
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);	$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);

function processFile($fpath) {	function processFile($fpath) {
global $conn, $contractNoticeFields, $contractNoticeInsertQ;	global $conn, $contractNoticeFields, $contractNoticeInsertQ;
$row = 1;	$row = 1;
$handle = fopen($fpath, "r");	$handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix	//"t" mode string translates windows line breaks to unix
$datamapping0712 = array(	$datamapping0712 = array(
"Agency" => "agencyName",	"Agency" => "agencyName",
"Parent CN ID" => "parentCN",	"Parent CN ID" => "parentCN",
"CN ID" => "CNID",	"CN ID" => "CNID",
"Publish Date" => "publishDate",	"Publish Date" => "publishDate",
"Amendment Date" => "amendDate",	"Amendment Date" => "amendDate",
"Status" => "",	"Status" => "",
"StartDate" => "contractStart",	"StartDate" => "contractStart",
"EndDate" => "contractEnd",	"EndDate" => "contractEnd",
"Value" => "value",	"Value" => "value",
"Description" => "description",	"Description" => "description",
"Agency Ref Id" => "agencyID",	"Agency Ref Id" => "agencyID",
"Agency Ref. ID" => "agencyID",	"Agency Ref. ID" => "agencyID",
"Category" => "category",	"Category" => "category",
"Procurement Method" => "procurementMethod",	"Procurement Method" => "procurementMethod",
"ATM ID" => "atmID",	"ATM ID" => "atmID",
"SON ID" => "SONID",	"SON ID" => "SONID",
"Confidentiality - Contract" => "confidentialityContract",	"Confidentiality - Contract" => "confidentialityContract",
"Confidentiality - Contract Reason(s)" => "confidentialityContractReason",	"Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
"Confidentiality - Outputs" => "confidentialityOutputs",	"Confidentiality - Outputs" => "confidentialityOutputs",
"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",	"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
"Consultancy" => "consultancy",	"Consultancy" => "consultancy",
"Consultancy Reason(s)" => "consultancyReason",	"Consultancy Reason(s)" => "consultancyReason",
"Amendment Reason" => "amendmentReason",	"Amendment Reason" => "amendmentReason",
"Supplier Name" => "supplierName",	"Supplier Name" => "supplierName",
"Supplier Address" => "supplierAddress",	"Supplier Address" => "supplierAddress",
"Supplier City" => "supplierCity",	"Supplier City" => "supplierCity",
"Supplier Postcode" => "supplierPostcode",	"Supplier Postcode" => "supplierPostcode",
"Supplier Country" => "supplierCountry",	"Supplier Country" => "supplierCountry",
"Supplier ABNExempt" => "supplierABNExempt",	"Supplier ABNExempt" => "supplierABNExempt",
"Supplier ABN" => "supplierABN",	"Supplier ABN" => "supplierABN",
"Agency Branch" => "contactBranch",	"Agency Branch" => "contactBranch",
"Agency Divison" => "contactDivision",	"Agency Divison" => "contactDivision",
"Agency Postcode" => "contactPostcode",	"Agency Postcode" => "contactPostcode",
"" => ""	"" => ""
);	);

$headers;

while (($data = fgetcsv($handle, 1000, "\t")) !== false) {	while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data);	$num = count($data);
if ($row == 3) {	if ($row == 3) {
$headers = $data;	$headers = $data;
} elseif ($row > 3) {	} elseif ($row > 3) {
if ($num > count($datamapping0712)) {	if ($num > count($datamapping0712)) {
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));	die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
}	}
$contractNoticeInsert = Array();	$contractNoticeInsert = Array();
$supplierInsert = Array();	$supplierInsert = Array();
$agencyInsert = Array();	$agencyInsert = Array();
$contractNoticeInsert[] = $fpath;	$contractNoticeInsert[] = $fpath;
$keys = array_keys($datamapping0712);	$keys = array_keys($datamapping0712);
for ($c = 0; $c < $num; $c++) {	for ($c = 0; $c < $num; $c++) {
$data[$c] = trim($data[$c], "=");	$data[$c] = trim($data[$c], "=");
$data[$c] = trim($data[$c], "\"");	$data[$c] = trim($data[$c], "\"");
if (in_array(($datamapping0712[$headers[$c]]), $contractNoticeFields)) {	if (in_array(($datamapping0712[$headers[$c]]), $contractNoticeFields)) {
if (($datamapping0712[$headers[$c]]) == "parentCN" \|\| ($datamapping0712[$headers[$c]]) == "CNID") {	if (($datamapping0712[$headers[$c]]) == "parentCN" \|\| ($datamapping0712[$headers[$c]]) == "CNID") {
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix	$data[$c] = substr($data[$c], 2); // take off the "CN" prefix
if ($data[$c] > 0 && $data[$c] != '0') {	if ($data[$c] > 0 && $data[$c] != '0') {
$contractNoticeInsert[] = $data[$c];	$contractNoticeInsert[] = $data[$c];
} else {	} else {
$contractNoticeInsert[] = null;	$contractNoticeInsert[] = null;
}	}
} elseif (($datamapping0712[$headers[$c]]) == "supplierABN") {	} elseif (($datamapping0712[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0 && $data[$c] != '0') {	if ($data[$c] > 0 && $data[$c] != '0') {
$contractNoticeInsert[] = $data[$c];	$contractNoticeInsert[] = $data[$c];
} else {	} else {
$contractNoticeInsert[] = null;	$contractNoticeInsert[] = null;
}	}
} elseif (($datamapping0712[$headers[$c]]) == "amendDate" \|\| ($datamapping0712[$headers[$c]]) == "publishDate" \|\| ($datamapping0712[$headers[$c]]) == "contractStart" \|\| ($datamapping0712[$headers[$c]]) == "contractEnd") {	} elseif (($datamapping0712[$headers[$c]]) == "amendDate" \|\| ($datamapping0712[$headers[$c]]) == "publishDate" \|\| ($datamapping0712[$headers[$c]]) == "contractStart" \|\| ($datamapping0712[$headers[$c]]) == "contractEnd") {
$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));	$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
} else {	} else {
if (strstr("\" =", $data[$c] > 0)) {	if (strstr("\" =", $data[$c] > 0)) {
die("Invalid Description field" . $contractNoticeInsert);	die("Invalid Description field" . $contractNoticeInsert);
}	}
$colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));	$colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));

$contractNoticeInsert[] = $colvalue;	$contractNoticeInsert[] = $colvalue;
}	}
}	}
}	}
flush();	flush();
$contractNoticeInsertQ->execute($contractNoticeInsert);	$contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo();	$errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {	if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {

} elseif ($errors[1] == 0) {	} elseif ($errors[1] == 0) {
$success++;	$success++;
} else {	} else {
foreach ($contractNoticeFields as $key => $cnf) {	foreach ($contractNoticeFields as $key => $cnf) {
echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";	echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
}	}
echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";	echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
}	}

flush();	flush();
//echo "<hr>\n";	//echo "<hr>\n";
}	}
$row++;	$row++;
}	}
fclose($handle);	fclose($handle);
$contractNoticeInsertQ->closeCursor();	$contractNoticeInsertQ->closeCursor();

return $success;	return $success;
}	}

$path = 'data/';	$path = 'data/';
if ($_REQUEST["fname"] == "") {	if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path);	$dhandle = opendir($path);
// define an array to hold the files	// define an array to hold the files
$files = array();	$files = array();
if ($dhandle) {	if ($dhandle) {
// loop through all of the files	// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {	while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) {	if (($fname != '.') && ($fname != '..')) {
$files[date("c", filemtime($path . $fname)) . md5($fname)] = $fname;	$files[date("c", filemtime($path . $fname)) . md5($fname)] = $fname;
}	}
}	}
}	}
ksort($files);	ksort($files);
foreach ($files as $date => $fname) {	foreach ($files as $date => $fname) {
echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";	echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
}	}
} else {	} else {
$success = 0;	$success = 0;
$fname = $_REQUEST["fname"];	$fname = $_REQUEST["fname"];
echo " ============== $fname ============== <br>";	echo " ============== $fname ============== <br>";
flush();	flush();
$success+= processFile($path . $fname, "contractnotice");	$success+= processFile($path . $fname, "contractnotice");
$success+= processFile($path . $fname, "agency");	$success+= processFile($path . $fname, "agency");
$success+= processFile($path . $fname, "supplier");	$success+= processFile($path . $fname, "supplier");
echo "<br> $success records successfully created";	echo "<br> $success records successfully created";

flush();	flush();
// run post import data processing	// run post import data processing
//	//
if ($success > 0) {	if ($success > 0) {
$conn->exec("update datasets set \"lastUpdated\" = NOW() where title = 'Contract Notices'");	$conn->exec("update datasets set \"lastUpdated\" = NOW() where title = 'Contract Notices'");
echo "link amend<br>";	echo "link amend<br>";
include ("linkAmendments.php");	include ("linkAmendments.php");
echo "update UNSPSC<br>";	echo "update UNSPSC<br>";
include ("updateUNSPSC.php");	include ("updateUNSPSC.php");
}	}
// cn	// cn

// agency	// agency
//include ("setAgencyStatus.php");	//include ("setAgencyStatus.php");
//include ("setAgencyURLABN.php");	//include ("setAgencyURLABN.php");
}	}
}	}
?>	?>

file:b/admin/importdatagov.php (new)

	<?php

	include_once ("../lib/common.inc.php");
	auth();
	/* todo gaps
	Contract ID,Department,Portfolio,Division,Branch,Agency Ref,Contract Date,End Date,Value,Office PCode,Description,Procurement Desc,Confidentiality Reason,Consultancy,ANZSCC Code,ANSCC Desc,Supplier,Supplier Postal Addr,Supplier Suburb,Supplier State,Supplier Country,Supplier PCode,Supplier ABN,Supplier DUNS,Supplier ACN,S/O Reference Num
	1694570,Department of Defence,Defence,NEW SOUTH WALES,RAN HMAS WATSON,1906458734 ,30-Jun-2007,30-Jun-2007,16083.96,2612,AIRLINE TICKETS,direct source,,No,731,Passenger transportation by air,QANTAS AIRWAYS LTD,PO Box PB 747,MASCOT,NSW,Australia,2020,16009661901,750512642,009661901 ,

	*/
	$contractNoticeFields = array(
	"importFile",
	"agencyName",
	"parentCN",
	"CNID",
	"publishDate",
	"amendDate",
	"contractStart",
	"contractEnd",
	"value",
	"description",
	"agencyID",
	"categoryUNSPSC",
	"category",
	"procurementMethod",
	"atmID",
	"SONID",
	"confidentialityContract",
	"confidentialityContractReason",
	"confidentialityOutputs",
	"confidentialityOutputsReason",
	"consultancy",
	"consultancyReason",
	"amendmentReason",
	"supplierName",
	"supplierAddress",
	"supplierCity",
	"supplierPostcode",
	"supplierCountry",
	"supplierABNExempt",
	"supplierABN",
	"contactBranch",
	"contactDivision",
	"contactPostcode"
	);

	$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
	foreach ($contractNoticeFields as $key => $f) {
	$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
	}
	$contractNoticeInsertQ.= ");";
	$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);

	function processFile($fpath) {
	global $conn, $contractNoticeFields, $contractNoticeInsertQ;
	$row = 1;
	$success = 0;
	ini_set('auto_detect_line_endings',TRUE);
	$handle = fopen($fpath, "r");
	//"t" mode string translates windows line breaks to unix
	$datamapping0712 = array(

	"Agency Name" => "agencyName",
	"Parent Contract ID" => "parentCN",
	"Contract ID" => "CNID",
	"Publish Date" => "publishDate",
	"Amendment Date" => "amendDate",
	"Start Date" => "contractStart",
	"End Date" => "contractEnd",
	"Value" => "value",
	"Description" => "description",
	"Agency Ref ID" => "agencyID",
	"UNSPSC Code" => "categoryUNSPSC",
	"Title" => "category",
	"Procurement Method" => "procurementMethod",
	"ATM ID" => "atmID",
	"SON ID" => "SONID",
	"Confidentiality Contract Flag" => "confidentialityContract",
	"Confidentiality Contract Reason" => "confidentialityContractReason",
	"Confidentiality Outputs Flag" => "confidentialityOutputs",
	"Confidentiality Outputs Reason" => "confidentialityOutputsReason",
	"Consultancy Flag" => "consultancy",
	"Consultancy Reason" => "consultancyReason",
	"Amendment Reason" => "amendmentReason",
	"Supplier Name" => "supplierName",
	"Supplier Address" => "supplierAddress",
	"Supplier Suburb" => "supplierCity",
	"Supplier Postcode" => "supplierPostcode",
	"Supplier Country" => "supplierCountry",
	"Supplier ABN Exempt" => "supplierABNExempt",
	"ABN" => "supplierABN",
	"Contact Name" => "",
	"Contact Phone" => "",
	"Branch" => "contactBranch",
	"Division" => "contactDivision",
	"Office Postcode" => "contactPostcode",

	);


	while (($data = fgetcsv($handle, 10000)) !== false) {
	//print_r($data);
	$num = count($data);
	if ($row == 1) {
	$headers = $data;
	} elseif ($row > 1) {
	if ($num > count($datamapping0712)) {
	die("<font color=red>Error in data import; data mapping fields out of bounds or changed $num > ".count($datamapping0712)."</font><br>" . $fpath . print_r($data));
	}
	$contractNoticeInsert = Array();
	$supplierInsert = Array();
	$agencyInsert = Array();
	$contractNoticeInsert[] = $fpath;
	$keys = array_keys($datamapping0712);
	for ($c = 0; $c < $num; $c++) {
	$data[$c] = trim($data[$c], "=");
	$data[$c] = trim($data[$c], "\"");
	if (in_array(($datamapping0712[$headers[$c]]), $contractNoticeFields)) {
	if (($datamapping0712[$headers[$c]]) == "parentCN" \|\| ($datamapping0712[$headers[$c]]) == "CNID") {
	$data[$c] = substr($data[$c], 2); // take off the "CN" prefix
	if ($data[$c] > 0 && $data[$c] != '0') {
	$contractNoticeInsert[] = $data[$c];
	} else {
	$contractNoticeInsert[] = null;
	}
	} elseif (($datamapping0712[$headers[$c]]) == "supplierABN") {
	if ($data[$c] > 0 && $data[$c] != '0') {
	$contractNoticeInsert[] = $data[$c];
	} else {
	$contractNoticeInsert[] = null;
	}
	} elseif (($datamapping0712[$headers[$c]]) == "amendDate" \|\| ($datamapping0712[$headers[$c]]) == "publishDate" \|\| ($datamapping0712[$headers[$c]]) == "contractStart" \|\| ($datamapping0712[$headers[$c]]) == "contractEnd") {
	$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
	} else {
	if (strstr("\" =", $data[$c] > 0)) {
	die("Invalid Description field" . $contractNoticeInsert);
	}
	$colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));

	$contractNoticeInsert[] = $colvalue;
	}
	}
	}
	flush();
	//print_r($contractNoticeInsert);
	$contractNoticeInsertQ->execute($contractNoticeInsert);
	$errors = $conn->errorInfo();
	if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
	// echo "dupe <br>";
	} elseif ($errors[1] == 0) {
	$success++;
	} else {
	foreach ($contractNoticeFields as $key => $cnf) {
	echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
	}
	echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
	}

	flush();
	//echo "<hr>\n";
	}
	$row++;
	}
	fclose($handle);
	$contractNoticeInsertQ->closeCursor();

	return $success;
	}

	$path = 'datagovdata/';
	if ($_REQUEST["fname"] == "" && $argv[1] == "") {
	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
	$dhandle = opendir($path);
	// define an array to hold the files
	$files = array();
	if ($dhandle) {
	// loop through all of the files
	while (false !== ($fname = readdir($dhandle))) {
	if (($fname != '.') && ($fname != '..')) {
	$files[date("c", filemtime($path . $fname)) . md5($fname)] = $fname;
	}
	}
	}
	ksort($files);
	foreach ($files as $date => $fname) {
	echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
	}
	} else {
	$success = 0;
	$fname = $_REQUEST["fname"];
	if ($argv[1] != "") $fname = $argv[1];
	echo " ============== $fname ============== <br>";
	flush();
	$success+= processFile($path . $fname, "contractnotice");
	$success+= processFile($path . $fname, "agency");
	$success+= processFile($path . $fname, "supplier");
	echo "<br> $success records successfully created";

	flush();
	// run post import data processing
	//
	if ($success > 0) {
	$conn->exec("update datasets set \"lastUpdated\" = NOW() where title = 'Contract Notices'");
	echo "link amend<br>";
	include ("linkAmendments.php");
	echo "update UNSPSC<br>";
	include ("updateUNSPSC.php");
	}
	// cn

	// agency
	//include ("setAgencyStatus.php");
	//include ("setAgencyURLABN.php");

	}
	?>

file:a/admin/partialdata/import.php -> file:b/admin/partialdata/import.php

<?php	<?php
if (php_sapi_name() == "cli") {	if (php_sapi_name() == "cli") {
include_once ("../../lib/common.inc.php");	include_once ("../../lib/common.inc.php");
function processFile($fpath, $tablename)	function processFile($fpath, $tablename)
{	{
global $conn;	global $conn;
echo " ============== $fpath ============== <br>";	echo " ============== $fpath ============== <br>";
flush();	flush();
$row = 1;	$row = 1;
$success = 0;	$success = 0;
$dupes = 0;	$dupes = 0;
$handle = fopen($fpath, "r");	$handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix	//"t" mode string translates windows line breaks to unix
$datamapping0507 = array(	$datamapping0507 = array(

"Agency" => "agencyName",	"Agency" => "agencyName",
"CN ID" => "CNID",	"CN ID" => "CNID",
"Publish Date" => "publishDate",	"Publish Date" => "publishDate",
"Contract Start Date" => "contractStart",	"Contract Start Date" => "contractStart",
"Contract End Date" => "contractEnd",	"Contract End Date" => "contractEnd",
"Value (AUD)" => "value",	"Value (AUD)" => "value",
"Title" => "description",	"Title" => "description",
"Category" => "category",	"Category" => "category",
"ATM ID" => "atmID",	"ATM ID" => "atmID",
"Supplier Name" => "supplierName",	"Supplier Name" => "supplierName",
"LastUpdated" => "amendDate",	"LastUpdated" => "amendDate",
"" => ""	"" => ""
);	);
$headers;	$headers;
$contractNoticeFields = array(	$contractNoticeFields = array(
"importFile",	"importFile",
"CNID",	"CNID",
"description",	"description",
"agencyName",	"agencyName",
"publishDate",	"publishDate",
"category",	"category",
"contractStart",	"contractStart",
"contractEnd",	"contractEnd",
"value",	"value",
"atmID",	"atmID",
"supplierName",	"supplierName",
"amendDate"	"amendDate"
);	);
if ($tablename == "contractnotice") {	if ($tablename == "contractnotice") {
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';	$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
foreach ($contractNoticeFields as $key => $f) {	foreach ($contractNoticeFields as $key => $f) {
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";	$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
}	}
$contractNoticeInsertQ.= ");";	$contractNoticeInsertQ.= ");";
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);	$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
}	}

while (($data = fgetcsv($handle, 1000, "\t")) !== false) {	while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data);	$num = count($data);
if ($row == 3) {	if ($row == 3) {
$headers = $data;	$headers = $data;
}	}
elseif ($row > 3) {	elseif ($row > 3) {
if ($num > count($datamapping0507)) {	if ($num > count($datamapping0507)) {
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . "data:" .$num. print_r($data ,true). "mapping:" . count($datamapping0507). print_r($datamapping0507 ,true));	die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . "data:" .$num. print_r($data ,true). "mapping:" . count($datamapping0507). print_r($datamapping0507 ,true));
}	}
$contractNoticeInsert = Array();	$contractNoticeInsert = Array();
$contractNoticeInsert[] = $fpath;	$contractNoticeInsert[] = $fpath;
$keys = array_keys($datamapping0507);	$keys = array_keys($datamapping0507);
for ($c = 0; $c < $num; $c++) {	for ($c = 0; $c < $num; $c++) {
$data[$c] = trim($data[$c], "=");	$data[$c] = trim($data[$c], "=");
$data[$c] = trim($data[$c], "\"");	$data[$c] = trim($data[$c], "\"");
if ($tablename == "contractnotice") {	if ($tablename == "contractnotice") {
if (in_array(($datamapping0507[$headers[$c]]) , $contractNoticeFields)) {	if (in_array(($datamapping0507[$headers[$c]]) , $contractNoticeFields)) {
if (($datamapping0507[$headers[$c]]) == "parentCN" \|\| ($datamapping0507[$headers[$c]]) == "CNID") {	if (($datamapping0507[$headers[$c]]) == "parentCN" \|\| ($datamapping0507[$headers[$c]]) == "CNID") {
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix	$data[$c] = substr($data[$c], 2); // take off the "CN" prefix
if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric");	if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric");
if ($data[$c] > 0) {	if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c];	$contractNoticeInsert[] = $data[$c];
}	}
else {	else {
$contractNoticeInsert[] = 0;	$contractNoticeInsert[] = 0;
}	}
}	}
elseif (($datamapping0507[$headers[$c]]) == "supplierABN") {	elseif (($datamapping0507[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0) {	if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c];	$contractNoticeInsert[] = $data[$c];
}	}
else {	else {
$contractNoticeInsert[] = null;	$contractNoticeInsert[] = null;
}	}
}	}
elseif (($datamapping0507[$headers[$c]]) == "amendDate" \|\| ($datamapping0507[$headers[$c]]) == "publishDate" \|\| ($datamapping0507[$headers[$c]]) == "contractStart" \|\| ($datamapping0507[$headers[$c]]) == "contractEnd") {	elseif (($datamapping0507[$headers[$c]]) == "amendDate" \|\| ($datamapping0507[$headers[$c]]) == "publishDate" \|\| ($datamapping0507[$headers[$c]]) == "contractStart" \|\| ($datamapping0507[$headers[$c]]) == "contractEnd") {
$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));	$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
}	}
else {	else {
if (strstr("\" =", $data[$c] > 0)) {	if (strstr("\" =", $data[$c] > 0)) {
die("Invalid Description field" . $contractNoticeInsert);	die("Invalid Description field" . $contractNoticeInsert);
}	}
$colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));	$colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));

$contractNoticeInsert[] = $colvalue;	$contractNoticeInsert[] = $colvalue;
}	}
}	}
}	}
}	}
flush();	flush();
if ($tablename == "contractnotice") {	if ($tablename == "contractnotice") {
$contractNoticeInsertQ->execute($contractNoticeInsert);	$contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo();	$errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {	if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
$dupes++;	$dupes++;
}	}
elseif ($errors[1] == 0) {	elseif ($errors[1] == 0) {
$success++;	$success++;
}	}
else {	else {
foreach ($contractNoticeFields as $key => $cnf) {	foreach ($contractNoticeFields as $key => $cnf) {
echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";	echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
}	}
echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";	echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
}	}
}	}
flush();	flush();
//echo "<hr>\n";	//echo "<hr>\n";

}	}
$row++;	$row++;
}	}
fclose($handle);	fclose($handle);
echo " $dupes duplicate records<br>";	echo " $dupes duplicate records<br>";
echo " $success records successfully created<br>";	echo " $success records successfully created<br>";
flush();	flush();
return $success;	return $success;
}	}
$path = './';	$path = './';
if ($_REQUEST["fname"] == "") {	if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path);	$dhandle = opendir($path);
// define an array to hold the files	// define an array to hold the files
$files = array();	$files = array();
if ($dhandle) {	if ($dhandle) {
// loop through all of the files	// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {	while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..') && (!isset($_REQUEST["filter"]) \|\| strpos($fname,$_REQUEST["filter"]) != false)) {	if (($fname != '.') && ($fname != '..') && (!isset($_REQUEST["filter"]) \|\| strpos($fname,$_REQUEST["filter"]) != false)) {
echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";	echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice");	processFile($path . $fname, "contractnotice");
}	}
}	}
}	}
}	}
else {	else {
$success = 0;	$success = 0;
$fname = $_REQUEST["fname"];	$fname = $_REQUEST["fname"];

$success+= processFile($path . $fname, "contractnotice");	$success+= processFile($path . $fname, "contractnotice");

}	}
}	}
?>	?>

file:a/admin/partialdata/importamendments.php -> file:b/admin/partialdata/importamendments.php

file:a/heuristics/historyHeuristics.php -> file:b/heuristics/historyHeuristics.php

file:a/lib/common.inc.php -> file:b/lib/common.inc.php

<?php	<?php
date_default_timezone_set("Australia/ACT");	date_default_timezone_set("Australia/ACT");

error_reporting(E_ALL ^ E_NOTICE);	error_reporting(E_ALL ^ E_NOTICE);


$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");	$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");

if (!$conn) {	if (!$conn) {
die("A database error occurred.\n");	die("A database error occurred.\n");
}	}

define('ROOT', pathinfo(__FILE__, PATHINFO_DIRNAME));	define('ROOT', pathinfo(__FILE__, PATHINFO_DIRNAME));
if (strstr($_SERVER['PHP_SELF'], "labs/") \|\| strstr($_SERVER['PHP_SELF'], "admin/") \|\| strstr($_SERVER['PHP_SELF'], "heuristics/")) {	if (strstr($_SERVER['PHP_SELF'], "labs/") \|\| strstr($_SERVER['PHP_SELF'], "admin/") \|\| strstr($_SERVER['PHP_SELF'], "heuristics/")) {
$basePath = "../";	$basePath = "../";
}	}
	if (php_sapi_name() != "cli") {

require $basePath."lib/amon-php/amon.php";	require $basePath."lib/amon-php/amon.php";
Amon::config(array('address'=> 'http://127.0.0.1:2464',	Amon::config(array('address'=> 'http://127.0.0.1:2464',
'protocol' => 'http',	'protocol' => 'http',
'secret_key' => "JBcSUdFOi5lK0vCjLjbHDpQamcBnRA4iV7QLaTADeDQ"));	'secret_key' => "JBcSUdFOi5lK0vCjLjbHDpQamcBnRA4iV7QLaTADeDQ"));
Amon::setup_exception_handler();	Amon::setup_exception_handler();
	}

require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php';	require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php';
$openid = new LightOpenID($_SERVER['HTTP_HOST']);	$openid = new LightOpenID($_SERVER['HTTP_HOST']);
// you have to open the session to be able to modify or remove it	// you have to open the session to be able to modify or remove it
session_start();	session_start();
function login() {	function login() {
global $openid;	global $openid;
if (!$openid->mode) {	if (!$openid->mode) {
$openid->required = array('contact/email');	$openid->required = array('contact/email');
$openid->identity = 'https://www.google.com/accounts/o8/id';	$openid->identity = 'https://www.google.com/accounts/o8/id';
header('Location: ' . $openid->authUrl());	header('Location: ' . $openid->authUrl());
}	}
}	}

function auth() {	function auth() {
global $openid;	global $openid;
if ($_SESSION['authed'] == true) {	if ($_SESSION['authed'] == true) {
return true;	return true;
}	}

if ($openid->mode) {	if ($openid->mode) {
$attr = $openid->getAttributes();	$attr = $openid->getAttributes();
if ($attr['contact/email'] != 'maxious@gmail.com') {	if ($attr['contact/email'] != 'maxious@gmail.com') {
die('Access Denied');	die('Access Denied');
} else {	} else {
$_SESSION['authed'] = true;	$_SESSION['authed'] = true;
}	}
} else {	} else {
login();	login();
}	}
}	}

// $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);	// $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
function databaseError($errMsg) {	function databaseError($errMsg) {
if ($errMsg[2] != "") {	if ($errMsg[2] != "") {
echo '<div class="alert-message error">';	echo '<div class="alert-message error">';
Amon::log(print_r($errMsg, true).print_r($_REQUEST, true).print_r($_SERVER, true), array('error'));	Amon::log(print_r($errMsg, true).print_r($_REQUEST, true).print_r($_SERVER, true), array('error'));
die(print_r($errMsg, true));	die(print_r($errMsg, true));
echo "</div>";	echo "</div>";
}	}
}	}

function ucsmart($str) {	function ucsmart($str) {
$shortWords = Array("The", "Pty", "Ltd", "Inc", "Red", "Oil", "A", "An", "And", "At", "For", "In"	$shortWords = Array("The", "Pty", "Ltd", "Inc", "Red", "Oil", "A", "An", "And", "At", "For", "In"
, "Of", "On", "Or", "The", "To", "With");	, "Of", "On", "Or", "The", "To", "With");
$strArray = explode(" ", preg_replace("/(?<=(?<!:\|’s)\W)	$strArray = explode(" ", preg_replace("/(?<=(?<!:\|’s)\W)
(A\|An\|And\|At\|For\|In\|Of\|On\|Or\|The\|To\|With)	(A\|An\|And\|At\|For\|In\|Of\|On\|Or\|The\|To\|With)
(?=\W)/e", 'strtolower("$1")', ucwords(strtolower($str))));	(?=\W)/e", 'strtolower("$1")', ucwords(strtolower($str))));
foreach ($strArray as &$word) {	foreach ($strArray as &$word) {
if (strlen($word) <= 4 && !in_array($word, $shortWords))	if (strlen($word) <= 4 && !in_array($word, $shortWords))
$word = strtoupper($word);	$word = strtoupper($word);
}	}
return implode(" ", $strArray);	return implode(" ", $strArray);
}	}

function percent($num_amount, $num_total) {	function percent($num_amount, $num_total) {
$count1 = $num_amount / $num_total;	$count1 = $num_amount / $num_total;
$count2 = $count1 * 100;	$count2 = $count1 * 100;
$count = number_format($count2, 2);	$count = number_format($count2, 2);
return $count;	return $count;
}	}

function array_sum_all($a) {	function array_sum_all($a) {
if (!is_array($a))	if (!is_array($a))
return $a;	return $a;
foreach ($a as $key => $value)	foreach ($a as $key => $value)
$totale += array_sum_all($value);	$totale += array_sum_all($value);
return $totale;	return $totale;
}	}

// magic query modifiers	// magic query modifiers
$agency = filter_var($_REQUEST['agency'], FILTER_SANITIZE_STRING);	$agency = filter_var($_REQUEST['agency'], FILTER_SANITIZE_STRING);
if ($agency != "")	if ($agency != "")
$agencyQ = "agencyName = '" . $agency . "' AND ";	$agencyQ = "agencyName = '" . $agency . "' AND ";

$supplier = filter_var($_REQUEST['supplier'], FILTER_SANITIZE_STRING);	$supplier = filter_var($_REQUEST['supplier'], FILTER_SANITIZE_STRING);
if ($supplier != "") {	if ($supplier != "") {
$supplierParts = explode("-", $supplier);	$supplierParts = explode("-", $supplier);
$supplierName = "%" . $supplierParts[1] . "%";	$supplierName = "%" . $supplierParts[1] . "%";
$supplierABN = $supplierParts[0];	$supplierABN = $supplierParts[0];
if ($supplierParts[0] > 0)	if ($supplierParts[0] > 0)
$supplierQ = ' "supplierABN" = :supplierABN AND ';	$supplierQ = ' "supplierABN" = :supplierABN AND ';
else	else
$supplierQ = ' "supplierName" LIKE :supplierName AND ';	$supplierQ = ' "supplierName" LIKE :supplierName AND ';
}	}

$startYear = 2007;	$startYear = 2007;
$year = filter_var($_REQUEST['year'], FILTER_SANITIZE_NUMBER_INT);	$year = filter_var($_REQUEST['year'], FILTER_SANITIZE_NUMBER_INT);
if ($year != "") {	if ($year != "") {
$yearQ = "YEAR(publishDate) = " . $year . " AND ";	$yearQ = "YEAR(publishDate) = " . $year . " AND ";
}	}
$standardQ = ' "childCN" is null '; // AND YEAR(contractStart) >= 2007 AND YEAR(contractStart) <= 2010';	$standardQ = ' "childCN" is null '; // AND YEAR(contractStart) >= 2007 AND YEAR(contractStart) <= 2010';
$start = 0.0;	$start = 0.0;

function local_url() {	function local_url() {
return "http://" . $_SERVER['HTTP_HOST'] . rtrim(dirname($_SERVER['PHP_SELF']), '/\\') . "/";	return "http://" . $_SERVER['HTTP_HOST'] . rtrim(dirname($_SERVER['PHP_SELF']), '/\\') . "/";
}	}

function include_header($title) {	function include_header($title) {
global $start;	global $start;
?>	?>
<!DOCTYPE html>	<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" version="XHTML+RDFa 1.1"	<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" version="XHTML+RDFa 1.1"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"	xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"	xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:gr="http://purl.org/goodrelations/v1#"	xmlns:gr="http://purl.org/goodrelations/v1#"
xmlns:dc="http://purl.org/dc/terms/"	xmlns:dc="http://purl.org/dc/terms/"
xmlns:pc="http://purl.org/procurement#"	xmlns:pc="http://purl.org/procurement#"
xmlns:unspsc="http://www.ksl.stanford.edu/projects/DAML/UNSPSC.daml#"	xmlns:unspsc="http://www.ksl.stanford.edu/projects/DAML/UNSPSC.daml#"
xmlns:xsd="http://www.w3.org/2001/XMLSchema#"	xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
xmlns:pcdt="http://purl.org/procurement/public-contracts-datatypes#"	xmlns:pcdt="http://purl.org/procurement/public-contracts-datatypes#"
prefix="rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns#	prefix="rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns#
rdfs: http://www.w3.org/2000/01/rdf-schema#	rdfs: http://www.w3.org/2000/01/rdf-schema#
gr: http://purl.org/goodrelations/v1#	gr: http://purl.org/goodrelations/v1#
dcterms: http://purl.org/dc/terms/	dcterms: http://purl.org/dc/terms/
pc: http://purl.org/procurement/public-contracts#	pc: http://purl.org/procurement/public-contracts#
cpv: http://purl.org/weso/pscs/cpv/2008/resource/	cpv: http://purl.org/weso/pscs/cpv/2008/resource/
unspsc: http://www.ksl.stanford.edu/projects/DAML/UNSPSC.daml#	unspsc: http://www.ksl.stanford.edu/projects/DAML/UNSPSC.daml#
v: http://www.w3.org/2006/vcard/ns#	v: http://www.w3.org/2006/vcard/ns#
payment: http://reference.data.gov.uk/def/payment#	payment: http://reference.data.gov.uk/def/payment#
br: http://purl.org/business-register#	br: http://purl.org/business-register#
xsd: http://www.w3.org/2001/XMLSchema#	xsd: http://www.w3.org/2001/XMLSchema#
pcdt: http://purl.org/procurement/public-contracts-datatypes#">	pcdt: http://purl.org/procurement/public-contracts-datatypes#">
<head>	<head>
<title><?php echo $title; ?> - Contract Dashboard</title>	<title><?php echo $title; ?> - Contract Dashboard</title>
<link rel="stylesheet" type="text/css" href="bootstrap.min.css">	<link rel="stylesheet" type="text/css" href="bootstrap.min.css">
<link rel="stylesheet" type="text/css" href="bootstrap-responsive.css">	<link rel="stylesheet" type="text/css" href="bootstrap-responsive.css">
<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->	<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
<!--[if lt IE 9]>	<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>	<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->	<![endif]-->
<script type="text/javascript" src="lib/bsn.AutoSuggest_2.1.3_comp.js" charset="utf-8"></script>	<script type="text/javascript" src="lib/bsn.AutoSuggest_2.1.3_comp.js" charset="utf-8"></script>
<link rel="stylesheet" href="autosuggest_inquisitor.css" type="text/css" media="screen" charset="utf-8" />	<link rel="stylesheet" href="autosuggest_inquisitor.css" type="text/css" media="screen" charset="utf-8" />
<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script>	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script>
<script type="text/javascript">	<script type="text/javascript">
$(document).ready(function()	$(document).ready(function()
{	{
//hide the all of the element with class msg_body	//hide the all of the element with class msg_body
$(".msg_body").hide();	$(".msg_body").hide();
//toggle the componenet with class msg_body	//toggle the componenet with class msg_body
$(".msg_head").click(function()	$(".msg_head").click(function()
{	{
$(this).next(".msg_body").slideToggle(600);	$(this).next(".msg_body").slideToggle(600);
});	});
});	});
</script>	</script>

<style type="text/css" title="currentStyle">	<style type="text/css" title="currentStyle">
@import "media/css/demo_table.css";	@import "media/css/demo_table.css";
</style>	</style>
<script type="text/javascript" language="javascript" src="media/js/jquery.dataTables.js"></script>	<script type="text/javascript" language="javascript" src="media/js/jquery.dataTables.js"></script>
<script type="text/javascript" language="javascript" src="lib/bootstrap-dropdown.js"></script>	<script type="text/javascript" language="javascript" src="lib/bootstrap-dropdown.js"></script>
<script type="text/javascript" charset="utf-8">	<script type="text/javascript" charset="utf-8">
jQuery.fn.dataTableExt.aTypes.unshift(	jQuery.fn.dataTableExt.aTypes.unshift(
function ( sData )	function ( sData )
{	{
var sValidChars = "0123456789.-,";	var sValidChars = "0123456789.-,";
var Char;	var Char;

/* Check the numeric part */	/* Check the numeric part */
for ( i=1 ; i<sData.length ; i++ )	for ( i=1 ; i<sData.length ; i++ )
{	{
Char = sData.charAt(i);	Char = sData.charAt(i);
if (sValidChars.indexOf(Char) == -1)	if (sValidChars.indexOf(Char) == -1)
{	{
return null;	return null;
}	}
}	}

/* Check prefixed by currency */	/* Check prefixed by currency */
if ( sData.charAt(0) == '$' \|\| sData.charAt(0) == '£' )	if ( sData.charAt(0) == '$' \|\| sData.charAt(0) == '£' )
{	{
return 'currency';	return 'currency';
}	}
return null;	return null;
}	}
);	);
jQuery.fn.dataTableExt.oSort['currency-asc'] = function(a,b) {	jQuery.fn.dataTableExt.oSort['currency-asc'] = function(a,b) {
/* Remove any commas (assumes that if present all strings will have a fixed number of d.p) */	/* Remove any commas (assumes that if present all strings will have a fixed number of d.p) */
var x = a == "-" ? 0 : a.replace( /,/g, "" );	var x = a == "-" ? 0 : a.replace( /,/g, "" );
var y = b == "-" ? 0 : b.replace( /,/g, "" );	var y = b == "-" ? 0 : b.replace( /,/g, "" );

/* Remove the currency sign */	/* Remove the currency sign */
x = x.substring( 1 );	x = x.substring( 1 );
y = y.substring( 1 );	y = y.substring( 1 );

/* Parse and return */	/* Parse and return */
x = parseFloat( x );	x = parseFloat( x );
y = parseFloat( y );	y = parseFloat( y );
return x - y;	return x - y;
};	};

jQuery.fn.dataTableExt.oSort['currency-desc'] = function(a,b) {	jQuery.fn.dataTableExt.oSort['currency-desc'] = function(a,b) {
/* Remove any commas (assumes that if present all strings will have a fixed number of d.p) */	/* Remove any commas (assumes that if present all strings will have a fixed number of d.p) */
var x = a == "-" ? 0 : a.replace( /,/g, "" );	var x = a == "-" ? 0 : a.replace( /,/g, "" );
var y = b == "-" ? 0 : b.replace( /,/g, "" );	var y = b == "-" ? 0 : b.replace( /,/g, "" );

/* Remove the currency sign */	/* Remove the currency sign */
x = x.substring( 1 );	x = x.substring( 1 );
y = y.substring( 1 );	y = y.substring( 1 );

/* Parse and return */	/* Parse and return */
x = parseFloat( x );	x = parseFloat( x );
y = parseFloat( y );	y = parseFloat( y );
return y - x;	return y - x;
};	};
$(document).ready(function() {	$(document).ready(function() {
$('table').dataTable();	$('table').dataTable();
} );	} );
</script>	</script>
<link type="text/css" rel="stylesheet" href="style.css">	<link type="text/css" rel="stylesheet" href="style.css">
</head>	</head>
<body>	<body>
<div class="navbar">	<div class="navbar">
<div class="navbar-inner">	<div class="navbar-inner">
<div class="container-fluid">	<div class="container-fluid">
<a class="brand" href="#">contract dashboard</a>	<a class="brand" href="#">contract dashboard</a>
<ul class="nav">	<ul class="nav">
<li><a href="displayAgency.php">agencies</a></li>	<li><a href="displayAgency.php">agencies</a></li>
<li><a href="displaySupplier.php">suppliers</a></li>	<li><a href="displaySupplier.php">suppliers</a></li>
<li><a href="displayCategory.php">categories</a></li>	<li><a href="displayCategory.php">categories</a></li>
<li><a href="displayCalendar.php">time periods</a></li>	<li><a href="displayCalendar.php">time periods</a></li>
<!-- <li class="dropdown">	<!-- <li class="dropdown">
<a href="#" class="dropdown-toggle">metrics</a>	<a href="#" class="dropdown-toggle">metrics</a>
<ul class="dropdown-menu">-->	<ul class="dropdown-menu">-->
<li><a href="displayProcurementMethod.php">tenderm</a></li>	<li><a href="displayProcurementMethod.php">tenderm</a></li>
<li><a href="displayConfidentialities.php">confidentiality</a></li>	<li><a href="displayConfidentialities.php">confidentiality</a></li>
<li><a href="displayConsultancies.php">consultancies</a></li>	<li><a href="displayConsultancies.php">consultancies</a></li>
<li><a href="displayAmendments.php">amendments</a></li>	<li><a href="displayAmendments.php">amendments</a></li>
<li><a href="displayMap.php">geo</a></li>	<li><a href="displayMap.php">geo</a></li>
</ul>	</ul>


<form method="post" action="search.php" class="pull-right">	<form method="post" action="search.php" class="pull-right">
<input type="text" id="searchKeyword" name="searchKeyword" value="" placeholder="Search" />	<input type="text" id="searchKeyword" name="searchKeyword" value="" placeholder="Search" />
<input type="hidden" id="searchID" name="searchID" value=""/>	<input type="hidden" id="searchID" name="searchID" value=""/>
</form>	</form>

</div>	</div>
</div><!-- /topbar-inner -->	</div><!-- /topbar-inner -->
</div><!-- /topbar -->	</div><!-- /topbar -->
</div><!-- /topbar-wrapper -->	</div><!-- /topbar-wrapper -->
<script type="text/javascript">	<script type="text/javascript">

var options_xml = {	var options_xml = {
script: function (input) { return "search_autosuggest.php?input="+input; },	script: function (input) { return "search_autosuggest.php?input="+input; },
varname:"input",	varname:"input",
callback: function (obj) { document.getElementById('searchID').value = obj.id; }	callback: function (obj) { document.getElementById('searchID').value = obj.id; }
};	};
var as_xml = new bsn.AutoSuggest('searchKeyword', options_xml);	var as_xml = new bsn.AutoSuggest('searchKeyword', options_xml);
</script>	</script>
<div class="container-fluid">	<div class="container-fluid">
<div class="row-fluid">	<div class="row-fluid">
<div class="span3">	<div class="span3">
<div class="well sidebar-nav">	<div class="well sidebar-nav">
<li class="nav-header">Filter by:</li>	<li class="nav-header">Filter by:</li>
<li>2008</li>	<li>2008</li>
</div>	</div>
</div>	</div>
<div class="span9">	<div class="span9">
<?php	<?php
$start = (float) array_sum(explode(' ', microtime()));	$start = (float) array_sum(explode(' ', microtime()));
}	}

function include_footer() {	function include_footer() {
global $start;	global $start;
$end = (float) array_sum(explode(' ', microtime()));	$end = (float) array_sum(explode(' ', microtime()));

echo '</div> <footer>' . "Processing time: " . sprintf("%.4f", ($end - $start)) . " seconds" . ' <footer>';	echo '</div> <footer>' . "Processing time: " . sprintf("%.4f", ($end - $start)) . " seconds" . ' <footer>';

if (strpos($_SERVER['SERVER_NAME'], ".gs")) {	if (strpos($_SERVER['SERVER_NAME'], ".gs")) {
?>	?>
<script type="text/javascript">	<script type="text/javascript">

var _gaq = _gaq \|\| [];	var _gaq = _gaq \|\| [];
_gaq.push(['_setAccount', 'UA-12341040-3']);	_gaq.push(['_setAccount', 'UA-12341040-3']);
_gaq.push(['_trackPageview']);	_gaq.push(['_trackPageview']);

(function() {	(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;	var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';	ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);	var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();	})();

</script>	</script>
<?php	<?php
}	}
echo '</div> </div></body> </html>';	echo '</div> </div></body> </html>';
}	}

include ("graphs.inc.php");	include ("graphs.inc.php");

<?php	<?php
$heuristics["HISTORY_LOW_TRANSACTIONS_AGENCY"] = Array(	$heuristics["HISTORY_LOW_TRANSACTIONS_AGENCY"] = Array(
"description" => "unusual for agency due to previous low number of transactions "	"description" => "unusual for agency due to previous low number of transactions "
);	);
function HISTORY_LOW_TRANSACTIONS_AGENCY($cn)	function HISTORY_LOW_TRANSACTIONS_AGENCY($cn)
{	{
$thisAgencyTransactions = getAgencyTransactions($cn['agencyName']);	$thisAgencyTransactions = getAgencyTransactions($cn['agencyName']);
$averageAgencyTransactions = getAverageAgencyTransactions();	$averageAgencyTransactions = getAverageAgencyTransactions();
$stddevAgencyTransactions = getstddevAgencyTransactions();	$stddevAgencyTransactions = getstddevAgencyTransactions();
$diff = strtotime($cn['contractEnd']) - strtotime($cn['contractStart']);	$diff = strtotime($cn['contractEnd']) - strtotime($cn['contractStart']);
$days = intval($diff / (60 * 60 * 24));	$days = intval($diff / (60 * 60 * 24));
$value = abs($days - $averageAgencyTransactions) / $stddevAgencyTransactions;	$value = abs($days - $averageAgencyTransactions) / $stddevAgencyTransactions;
return Array(	return Array(
"heuristic_value" => $value,	"heuristic_value" => $value,
"raw_value" => $days,	"raw_value" => $days,
"mean" => $averageAgencyTransactions,	"mean" => $averageAgencyTransactions,
"stddev" => $stddevAgencyTransactions	"stddev" => $stddevAgencyTransactions
);	);
}	}
$agencyTransactions = Array();	$agencyTransactions = Array();
function getAgencyTransactions($agencyName)	function getAgencyTransactions($agencyName)
{	{
global $agencyTransactions,$conn;	global $agencyTransactions,$conn;
if (!$agencyTransactions[$agencyName]) {	if (!$agencyTransactions[$agencyName]) {
$query = 'select count(*) from contractnotice where "agencyName" = \'' . pg_escape_string ($agencyName) . '"\'';	$query = 'select count(*) from contractnotice where "agencyName" = \'' . $agencyName . '"\'';
$result = $conn->query($query);	$result = $conn->query($query);
//echo $query;
$r = $result->fetch(PDO::FETCH_BOTH);	$r = $result->fetch(PDO::FETCH_BOTH);
$agencyTransactions[$agencyName] = $r[0];	$agencyTransactions[$agencyName] = $r[0];
}	}
return $agencyTransactions[$agencyName];	return $agencyTransactions[$agencyName];
}	}
$averageAgencyTransactions;	$averageAgencyTransactions;
function getAverageAgencyTransactions()	function getAverageAgencyTransactions()
{	{
global $averageAgencyTransactions;	global $averageAgencyTransactions;
if (!$averageAgencyTransactions) {	if (!$averageAgencyTransactions) {
getStatsAgencyTransactions();	getStatsAgencyTransactions();
}	}
return $averageAgencyTransactions;	return $averageAgencyTransactions;
}	}
$stddevAgencyTransactions;	$stddevAgencyTransactions;
function getstddevAgencyTransactions()	function getstddevAgencyTransactions()
{	{
global $stddevAgencyTransactions;	global $stddevAgencyTransactions;
if (!$stddevAgencyTransactions) {	if (!$stddevAgencyTransactions) {
getStatsAgencyTransactions();	getStatsAgencyTransactions();
}	}
return $stddevAgencyTransactions;	return $stddevAgencyTransactions;
}	}
function getStatsAgencyTransactions()	function getStatsAgencyTransactions()
{	{
global $averageAgencyTransactions, $stddevAgencyTransactions,$conn;	global $averageAgencyTransactions, $stddevAgencyTransactions,$conn;
$query = 'select avg(count), STDDEV(count) from (select count(*) as count	$query = 'select avg(count), STDDEV(count) from (select count(*) as count
from contractnotice group by "agencyName") as a;';	from contractnotice group by "agencyName") as a;';
$result = $conn->query($query);	$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);	$r = $result->fetch(PDO::FETCH_BOTH);
$averageAgencyTransactions = $r[0];	$averageAgencyTransactions = $r[0];
$stddevAgencyTransactions = $r[1];	$stddevAgencyTransactions = $r[1];
}	}
$heuristics["HISTORY_LOW_TRANSACTIONS_SUPPLIER"] = Array(	$heuristics["HISTORY_LOW_TRANSACTIONS_SUPPLIER"] = Array(
"description" => "unusual for supplier due to previous low number of transactions "	"description" => "unusual for supplier due to previous low number of transactions "
);	);
function HISTORY_LOW_TRANSACTIONS_SUPPLIER($cn)	function HISTORY_LOW_TRANSACTIONS_SUPPLIER($cn)
{	{
$thisSupplierTransactions = getSupplierTransactions($cn['supplierName'], $cn['supplierABN']);	$thisSupplierTransactions = getSupplierTransactions($cn['supplierName'], $cn['supplierABN']);
$averageSupplierTransactions = getAverageSupplierTransactions();	$averageSupplierTransactions = getAverageSupplierTransactions();
$stddevSupplierTransactions = getstddevSupplierTransactions();	$stddevSupplierTransactions = getstddevSupplierTransactions();
$diff = strtotime($cn['contractEnd']) - strtotime($cn['contractStart']);	$diff = strtotime($cn['contractEnd']) - strtotime($cn['contractStart']);
$days = intval($diff / (60 * 60 * 24));	$days = intval($diff / (60 * 60 * 24));
$value = abs($days - $averageSupplierTransactions) / $stddevSupplierTransactions;	$value = abs($days - $averageSupplierTransactions) / $stddevSupplierTransactions;
return Array(	return Array(
"heuristic_value" => $value,	"heuristic_value" => $value,
"raw_value" => $days,	"raw_value" => $days,
"mean" => $averageSupplierTransactions,	"mean" => $averageSupplierTransactions,
"stddev" => $stddevSupplierTransactions	"stddev" => $stddevSupplierTransactions
);	);
}	}
$supplierTransactions = Array();	$supplierTransactions = Array();
function getSupplierTransactions($supplierName, $supplierABN)	function getSupplierTransactions($supplierName, $supplierABN)
{	{
global $supplierTransactions,$conn;	global $supplierTransactions,$conn;
if ($supplierABN != 0 && $supplierABN != "") {	if ($supplierABN != 0 && $supplierABN != "") {
if (!$supplierTransactions[$supplierABN]) {	if (!$supplierTransactions[$supplierABN]) {
$query = "select count(*) from contractnotice where \"supplierABN\" = '" . $supplierABN . "'";	$query = "select count(*) from contractnotice where \"supplierABN\" = '" . $supplierABN . "'";
$result = $conn->query($query);	$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);	$r = $result->fetch(PDO::FETCH_BOTH);
$supplierTransactions[$supplierABN] = $r[0];	$supplierTransactions[$supplierABN] = $r[0];
}	}
return $supplierTransactions[$supplierABN];	return $supplierTransactions[$supplierABN];
}	}
if (!$supplierTransactions[$supplierName]) {	if (!$supplierTransactions[$supplierName]) {
$query = "select count(*) from contractnotice where \"supplierName\" = '" . $supplierName . "'";	$query = "select count(*) from contractnotice where \"supplierName\" = '" . $supplierName . "'";

$result = $conn->query($query);	$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);	$r = $result->fetch(PDO::FETCH_BOTH);
$supplierTransactions[$supplierName] = $r[0];	$supplierTransactions[$supplierName] = $r[0];
}	}
return $supplierTransactions[$supplierName];	return $supplierTransactions[$supplierName];
}	}
$averageSupplierTransactions;	$averageSupplierTransactions;
function getAverageSupplierTransactions()	function getAverageSupplierTransactions()
{	{
global $averageSupplierTransactions;	global $averageSupplierTransactions;
if (!$averageSupplierTransactions) {	if (!$averageSupplierTransactions) {
getStatsSupplierTransactions();	getStatsSupplierTransactions();
}	}
return $averageSupplierTransactions;	return $averageSupplierTransactions;
}	}
$stddevSupplierTransactions;	$stddevSupplierTransactions;
function getstddevSupplierTransactions()	function getstddevSupplierTransactions()
{	{
global $stddevSupplierTransactions;	global $stddevSupplierTransactions;
if (!$stddevSupplierTransactions) {	if (!$stddevSupplierTransactions) {
getStatsSupplierTransactions();	getStatsSupplierTransactions();
}	}
return $stddevSupplierTransactions;	return $stddevSupplierTransactions;
}	}
function getStatsSupplierTransactions()	function getStatsSupplierTransactions()
{	{
global $averageSupplierTransactions, $stddevSupplierTransactions,$conn;	global $averageSupplierTransactions, $stddevSupplierTransactions,$conn;
$query = 'select avg(count), stddev(count) from (	$query = 'select avg(count), stddev(count) from (
select (case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as "supplierID",	select (case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as "supplierID",
count(*) as count from contractnotice group by "supplierID") as a;';	count(*) as count from contractnotice group by "supplierID") as a;';
$result = $conn->query($query);	$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);	$r = $result->fetch(PDO::FETCH_BOTH);
$averageSupplierTransactions = $r[0];	$averageSupplierTransactions = $r[0];
$stddevSupplierTransactions = $r[1];	$stddevSupplierTransactions = $r[1];
}	}