projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Support partial data amendments with multi line descriptions
Support partial data amendments with multi line descriptions

file:a/admin/import.php -> file:b/admin/import.php
<?php <?php
include_once ("../lib/common.inc.php"); include_once ("../lib/common.inc.php");
function processFile($fpath, $tablename) function processFile($fpath, $tablename)
{ {
global $conn; global $conn;
$row = 1; $row = 1;
$handle = fopen($fpath, "r"); $handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix //"t" mode string translates windows line breaks to unix
$datamapping0711 = array( $datamapping0711 = array(
"Agency" => "agencyName", "Agency" => "agencyName",
"Parent CN ID" => "parentCN", "Parent CN ID" => "parentCN",
"CN ID" => "CNID", "CN ID" => "CNID",
"Publish Date" => "publishDate", "Publish Date" => "publishDate",
"Amendment Date" => "amendDate", "Amendment Date" => "amendDate",
"Status" => "", "Status" => "",
"StartDate" => "contractStart", "StartDate" => "contractStart",
"EndDate" => "contractEnd", "EndDate" => "contractEnd",
"Value" => "value", "Value" => "value",
"Description" => "description", "Description" => "description",
"Agency Ref Id" => "agencyID", "Agency Ref Id" => "agencyID",
"Category" => "category", "Category" => "category",
"Procurement Method" => "procurementMethod", "Procurement Method" => "procurementMethod",
"ATM ID" => "atmID", "ATM ID" => "atmID",
"SON ID" => "SONID", "SON ID" => "SONID",
"Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract" => "confidentialityContract",
"Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
"Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs" => "confidentialityOutputs",
"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
"Consultancy" => "consultancy", "Consultancy" => "consultancy",
"Consultancy Reason(s)" => "consultancyReason", "Consultancy Reason(s)" => "consultancyReason",
"Amendment Reason" => "amendmentReason", "Amendment Reason" => "amendmentReason",
"Supplier Name" => "supplierName", "Supplier Name" => "supplierName",
"Supplier Address" => "supplierAddress", "Supplier Address" => "supplierAddress",
"Supplier City" => "supplierCity", "Supplier City" => "supplierCity",
"Supplier Postcode" => "supplierPostcode", "Supplier Postcode" => "supplierPostcode",
"Supplier Country" => "supplierCountry", "Supplier Country" => "supplierCountry",
"Supplier ABNExempt" => "supplierABNExempt", "Supplier ABNExempt" => "supplierABNExempt",
"Supplier ABN" => "supplierABN", "Supplier ABN" => "supplierABN",
"Agency Branch" => "contactBranch", "Agency Branch" => "contactBranch",
"Agency Divison" => "contactDivision", "Agency Divison" => "contactDivision",
"Agency Postcode" => "contactPostcode", "Agency Postcode" => "contactPostcode",
"" => "" "" => ""
); );
$headers; $headers;
$contractNoticeFields = array( $contractNoticeFields = array(
"importFile", "importFile",
"agencyName", "agencyName",
"parentCN", "parentCN",
"CNID", "CNID",
"publishDate", "publishDate",
"amendDate", "amendDate",
"contractStart", "contractStart",
"contractEnd", "contractEnd",
"value", "value",
"description", "description",
"agencyID", "agencyID",
"category", "category",
"procurementMethod", "procurementMethod",
"atmID", "atmID",
"SONID", "SONID",
"confidentialityContract", "confidentialityContract",
"confidentialityContractReason", "confidentialityContractReason",
"confidentialityOutputs", "confidentialityOutputs",
"confidentialityOutputsReason", "confidentialityOutputsReason",
"consultancy", "consultancy",
"consultancyReason", "consultancyReason",
"amendmentReason", "amendmentReason",
"supplierName", "supplierName",
"supplierAddress", "supplierAddress",
"supplierCity", "supplierCity",
"supplierPostcode", "supplierPostcode",
"supplierCountry", "supplierCountry",
"supplierABNExempt", "supplierABNExempt",
"supplierABN", "supplierABN",
"contactBranch", "contactBranch",
"contactDivision", "contactDivision",
"contactPostcode" "contactPostcode"
); );
$agencyFields = array( $agencyFields = array(
"agencyName" "agencyName"
); );
$supplierFields = array( $supplierFields = array(
"supplierName", "supplierName",
"supplierAddress", "supplierAddress",
"supplierCity", "supplierCity",
"supplierPostcode", "supplierPostcode",
"supplierCountry", "supplierCountry",
"supplierABNExempt", "supplierABNExempt",
"supplierABN" "supplierABN"
); );
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( '; $contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
foreach ($contractNoticeFields as $key => $f) { foreach ($contractNoticeFields as $key => $f) {
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$contractNoticeInsertQ.= ");"; $contractNoticeInsertQ.= ");";
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( '; $supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
foreach ($supplierFields as $key => $f) { foreach ($supplierFields as $key => $f) {
$supplierInsertQ.= ($key == 0 ? "" : ", ") . "?"; $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$supplierInsertQ.= ");"; $supplierInsertQ.= ");";
$supplierInsertQ = $conn->prepare($supplierInsertQ); $supplierInsertQ = $conn->prepare($supplierInsertQ);
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
$agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( '; $agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
foreach ($agencyFields as $key => $f) { foreach ($agencyFields as $key => $f) {
$agencyInsertQ.= ($key == 0 ? "" : ", ") . "?"; $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
} }
$agencyInsertQ.= ");"; $agencyInsertQ.= ");";
$agencyInsertQ = $conn->prepare($agencyInsertQ); $agencyInsertQ = $conn->prepare($agencyInsertQ);
} }
while (($data = fgetcsv($handle, 1000, "\t")) !== false) { while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data); $num = count($data);
if ($row == 3) { if ($row == 3) {
$headers = $data; $headers = $data;
} }
elseif ($row > 3) { elseif ($row > 3) {
if ($num > count($datamapping0711)) { if ($num > count($datamapping0711)) {
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data)); die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
} }
$contractNoticeInsert = Array(); $contractNoticeInsert = Array();
$supplierInsert = Array(); $supplierInsert = Array();
$agencyInsert = Array(); $agencyInsert = Array();
$contractNoticeInsert[] = $fpath; $contractNoticeInsert[] = $fpath;
$keys = array_keys($datamapping0711); $keys = array_keys($datamapping0711);
for ($c = 0; $c < $num; $c++) { for ($c = 0; $c < $num; $c++) {
$data[$c] = trim($data[$c], "="); $data[$c] = trim($data[$c], "=");
$data[$c] = trim($data[$c], "\""); $data[$c] = trim($data[$c], "\"");
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
if (in_array(($datamapping0711[$headers[$c]]) , $contractNoticeFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $contractNoticeFields)) {
if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") { if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
$data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers $data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers
if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric"); if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric");
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = 0; $contractNoticeInsert[] = 0;
} }
} }
elseif (($datamapping0711[$headers[$c]]) == "supplierABN") { elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = null; $contractNoticeInsert[] = null;
} }
} }
elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") { elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c])); $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
} }
else { else {
if (strstr("\" =", $data[$c] > 0)) { if (strstr("\" =", $data[$c] > 0)) {
die("Invalid Description field" . $contractNoticeInsert); die("Invalid Description field" . $contractNoticeInsert);
} }
$colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c])); $colvalue = preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));
   
$contractNoticeInsert[] = $colvalue; $contractNoticeInsert[] = $colvalue;
} }
} }
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
if (in_array(($datamapping0711[$headers[$c]]) , $supplierFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $supplierFields)) {
if (($datamapping0711[$headers[$c]]) == "supplierABN") { if (($datamapping0711[$headers[$c]]) == "supplierABN") {
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
else { else {
$contractNoticeInsert[] = 0; $contractNoticeInsert[] = 0;
} }
} }
else { else {
$supplierInsert[] = $data[$c]; $supplierInsert[] = $data[$c];
} }
} }
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
if (in_array(($datamapping0711[$headers[$c]]) , $agencyFields)) { if (in_array(($datamapping0711[$headers[$c]]) , $agencyFields)) {
$agencyInsert[] = $data[$c]; $agencyInsert[] = $data[$c];
} }
} }
} }
flush(); flush();
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ->execute($contractNoticeInsert); $contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
foreach ($contractNoticeFields as $key => $cnf) { foreach ($contractNoticeFields as $key => $cnf) {
echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>"; echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
} }
echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n"; echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
} }
} }
else if ($tablename == "supplierdetails") { else if ($tablename == "supplierdetails") {
$supplierInsertQ->execute($supplierInsert); $supplierInsertQ->execute($supplierInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n"; echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
} }
} }
else if ($tablename == "agency") { else if ($tablename == "agency") {
$agencyInsertQ->execute($agencyInsert); $agencyInsertQ->execute($agencyInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
} }
elseif ($errors[1] == 0) { elseif ($errors[1] == 0) {
$success++; $success++;
} }
else { else {
echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n"; echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
} }
} }
flush(); flush();
//echo "<hr>\n"; //echo "<hr>\n";
} }
$row++; $row++;
} }
fclose($handle); fclose($handle);
return $success; return $success;
} }
$path = 'data/'; $path = 'data/';
if ($_REQUEST["fname"] == "") { if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>"; echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path); $dhandle = opendir($path);
// define an array to hold the files // define an array to hold the files
$files = array(); $files = array();
if ($dhandle) { if ($dhandle) {
// loop through all of the files // loop through all of the files
while (false !== ($fname = readdir($dhandle))) { while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) { if (($fname != '.') && ($fname != '..')) {
echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>"; echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
} }
} }
} }
} }
else { else {
$success = 0; $success = 0;
$fname = $_REQUEST["fname"]; $fname = $_REQUEST["fname"];
echo " ============== $fname ============== <br>"; echo " ============== $fname ============== <br>";
flush(); flush();
$success+= processFile($path . $fname, "contractnotice"); $success+= processFile($path . $fname, "contractnotice");
$success+= processFile($path . $fname, "agency"); $success+= processFile($path . $fname, "agency");
$success+= processFile($path . $fname, "supplier"); $success+= processFile($path . $fname, "supplier");
echo "<br> $success records successfully created"; echo "<br> $success records successfully created";
flush(); flush();
} }
// run post import data processing // run post import data processing
// cn // cn
//include ("linkAmendments.php"); include ("linkAmendments.php");
include ("updateUNSPSC.php"); include ("updateUNSPSC.php");
// agency // agency
include ("setAgencyStatus.php"); //include ("setAgencyStatus.php");
include ("setAgencyURLABN.php"); //include ("setAgencyURLABN.php");
?> ?>
   
file:a/admin/partialdata/importamendments.php -> file:b/admin/partialdata/importamendments.php
<?php <?php
include_once ("../../lib/common.inc.php"); include_once ("../../lib/common.inc.php");
/* /*
update contractnotice set "supplierABN" = a."supplierABN" update contractnotice set "supplierABN" = a."supplierABN"
from contractnotice as cn inner join (select "supplierABN", from contractnotice as cn inner join (select "supplierABN",
"supplierName" from contractnotice where "supplierABN" "supplierName" from contractnotice where "supplierABN"
IS NOT NULL and "supplierABN" != 0) as a on IS NOT NULL and "supplierABN" != 0) as a on
cn."supplierName" = a."supplierName" where cn."supplierName" = a."supplierName" where
cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN" cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN"
IS NULL or contractnotice."supplierABN" = 0) */ IS NULL or contractnotice."supplierABN" = 0) */
// http://www.lastcraft.com/browser_documentation.php // http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/ // http://code.google.com/p/phpquery/
require('phpQuery-onefile.php'); require('phpQuery-onefile.php');
function getURL($url) { function getURL($url) {
//return file_get_contents($url); //return file_get_contents($url);
$ch = curl_init($url); $ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 45); curl_setopt($ch, CURLOPT_TIMEOUT, 45);
  curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
  curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
$page = curl_exec($ch); $page = curl_exec($ch);
if (curl_errno($ch)) { if (curl_errno($ch)) {
echo "<font color=red> Database temporarily unavailable: "; echo "<font color=red> Database temporarily unavailable: ";
echo curl_errno($ch) . " " . curl_error($ch); echo curl_errno($ch) . " " . curl_error($ch);
echo $url; echo $url;
echo "</font><br>"; echo "</font><br>";
} }
curl_close($ch); curl_close($ch);
return $page; return $page;
} }
   
   
function getTextFromTHNode($Node, $Text = "") { function getTextFromTHNode($Node, $Text = "") {
if ($Node->tagName == null) if ($Node->tagName == null)
return $Text.$Node->textContent; return $Text.$Node->textContent;
if ($Node->tagName != "td") { if ($Node->tagName != "td") {
$Node = $Node->firstChild; $Node = $Node->firstChild;
if ($Node != null) if ($Node != null)
$Text = getTextFromTHNode($Node, $Text); $Text = getTextFromTHNode($Node, $Text);
   
while($Node->nextSibling != null) { while($Node->nextSibling != null) {
$Text = getTextFromTHNode($Node->nextSibling, $Text); $Text = getTextFromTHNode($Node->nextSibling, $Text);
$Node = $Node->nextSibling; $Node = $Node->nextSibling;
} }
} }
return $Text; return $Text;
} }
   
function getTextFromNode($Node, $Text = "") { function getTextFromNode($Node, $Text = "") {
if ($Node->tagName == null) if ($Node->tagName == null)
return $Text.$Node->textContent; return $Text.$Node->textContent;
if ($Node->tagName != "th" && $Node->tagName != "span") { if ($Node->tagName != "th" && $Node->tagName != "span") {
$Node = $Node->firstChild; $Node = $Node->firstChild;
if ($Node != null) if ($Node != null)
$Text = getTextFromNode($Node, $Text); $Text = getTextFromNode($Node, $Text);
   
while($Node->nextSibling != null) { while($Node->nextSibling != null) {
$Text = getTextFromNode($Node->nextSibling, $Text); $Text = getTextFromNode($Node->nextSibling, $Text);
$Node = $Node->nextSibling; $Node = $Node->nextSibling;
} }
} }
return $Text; return $Text;
} }
function dom_to_array($root) function dom_to_array($root)
{ {
$result = array(); $result = array();