projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Start graph and import refactoring
Start graph and import refactoring

file:a/about.php -> file:b/about.php
<?php <?php
include('./lib/common.inc.php'); include('./lib/common.inc.php');
include_header("Map"); include_header("Map");
?> ?>
<div class="msg_list"> <div class="msg_list">
<p class="msg_head">Header-1 </p> <p class="msg_head">Header-1 </p>
<div class="msg_body"> <div class="msg_body">
orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit
</div> </div>
<p class="msg_head">Header-2</p> <p class="msg_head">Header-2</p>
<div class="msg_body"> <div class="msg_body">
orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit
</div> </div>
<p class="msg_head">Header-3</p> <p class="msg_head">Header-3</p>
<div class="msg_body"> <div class="msg_body">
orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit orem ipsum dolor sit amet, consectetuer adipiscing elit
</div> </div>
</div> </div>
  <?php CNDistributionGraph(); ?>
  sourced from austender
sourced from austender  
Government Agency Information (2009-07-10): National Archives of Australia, http://www.naa.gov.au under Creative Commons - Attribution 2.5 Australia (CC-BY) Government Agency Information (2009-07-10): National Archives of Australia, http://www.naa.gov.au under Creative Commons - Attribution 2.5 Australia (CC-BY)
jpgraph for php  
ABR ABR
Redhat Liberation fonts  
   
<?php <?php
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
$query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end $query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
FROM contractnotice WHERE "childCN" = 0 AND "parentCN" = 0 AND DATE("importDate") = (select * from (SELECT DATE("importDate") FROM contractnotice WHERE "childCN" = 0 AND "parentCN" = 0 AND DATE("importDate") = (select * from (SELECT DATE("importDate")
FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) GROUP BY DATE("importDate")'; FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) GROUP BY DATE("importDate")';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} }
echo "<div>"; echo "<div>";
$stats = $query->fetch(PDO::FETCH_ASSOC); $stats = $query->fetch(PDO::FETCH_ASSOC);
echo $stats["count"] . " new records in period " . $stats["start"] . " to " . $stats["end"] ."<br>"; echo $stats["count"] . " new records in period " . $stats["start"] . " to " . $stats["end"] ."<br>";
$query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end $query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
FROM contractnotice WHERE ("childCN" != 0 OR "parentCN" != 0) AND DATE("importDate") = (select * from (SELECT DATE("importDate") FROM contractnotice WHERE ("childCN" != 0 OR "parentCN" != 0) AND DATE("importDate") = (select * from (SELECT DATE("importDate")
FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) GROUP BY DATE("importDate")'; FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) GROUP BY DATE("importDate")';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} }
echo "<div>"; echo "<div>";
$stats = $query->fetch(PDO::FETCH_ASSOC); $stats = $query->fetch(PDO::FETCH_ASSOC);
echo $stats["count"] . " updated records in period " . $stats["start"] . " to " . $stats["end"] ."<br>"; echo $stats["count"] . " updated records in period " . $stats["start"] . " to " . $stats["end"] ."<br>";
echo "Last updated: ". $stats["importday"]."<br>"; echo "Last updated: ". $stats["importday"]."<br>";
/* Check for null Procurement method and 0 ABN when not ABN exempt */ /* Check for null Procurement method and 0 ABN when not ABN exempt */
include_footer(); include_footer();
?> ?>
file:a/admin/displayUNSPSC.php (deleted)
<?php  
error_reporting(E_ALL);  
include_once("../lib/common.inc.php");  
 
 
$unspscresult= $conn->prepare('select * from "UNSPSCcategories";');  
$unspscresult->execute();  
foreach ($unspscresult->fetchAll() as $row) {  
$unspsc[$row['UNSPSC']] = $row['Title'];  
 
}  
$catsresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 2 ) as cat , SUM( "value" ) as value  
FROM contractnotice  
GROUP BY cat ;');  
echo "<table>";  
$catsresult->execute();  
foreach ($catsresult->fetchAll() as $row) {  
$catName = $unspsc[$row['cat']."0000000"].$row['cat'];  
if ($row['cat'] == "") $catName = "null";  
 
echo "<tr><td>$catName</td><td>".$row['value']."</td></tr>";  
}  
?>  
 
file:a/admin/import.php -> file:b/admin/import.php
<?php <?php
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
   
function processFile($fpath,$tablename){ function processFile($fpath,$tablename){
global $conn; global $conn;
$row = 1; $row = 1;
$handle = fopen($fpath, "r"); $handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix //"t" mode string translates windows line breaks to unix
$datamapping0711 = array("Agency" => "agencyName", "Parent CN ID" => "parentCN", "CN ID" => "CNID", "Publish Date" => "publishDate", "Amendment Date" => "amendDate", "Status" => "", "StartDate" => "contractStart", "EndDate" => "contractEnd", "Value" => "value", "Description" => "description", "Agency Ref Id" => "agencyID", "Category" => "category", "Procurement Method" => "procurementMethod", "ATM ID" => "atmID", "SON ID" => "SONID", "Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Consultancy" => "consultancy", "Consultancy Reason(s)" => "consultancyReason", "Amendment Reason" => "amendmentReason", "Supplier Name" => "supplierName", "Supplier Address" => "supplierAddress", "Supplier City" => "supplierCity", "Supplier Postcode" => "supplierPostcode", "Supplier Country" => "supplierCountry", "Supplier ABNExempt" => "supplierABNExempt", "Supplier ABN" => "supplierABN", "Agency Branch" => "contactBranch", "Agency Divison" => "contactDivision", "Agency Postcode" => "contactPostcode", "" => ""); $datamapping0711 = array("Agency" => "agencyName", "Parent CN ID" => "parentCN", "CN ID" => "CNID", "Publish Date" => "publishDate", "Amendment Date" => "amendDate", "Status" => "", "StartDate" => "contractStart", "EndDate" => "contractEnd", "Value" => "value", "Description" => "description", "Agency Ref Id" => "agencyID", "Category" => "category", "Procurement Method" => "procurementMethod", "ATM ID" => "atmID", "SON ID" => "SONID", "Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Consultancy" => "consultancy", "Consultancy Reason(s)" => "consultancyReason", "Amendment Reason" => "amendmentReason", "Supplier Name" => "supplierName", "Supplier Address" => "supplierAddress", "Supplier City" => "supplierCity", "Supplier Postcode" => "supplierPostcode", "Supplier Country" => "supplierCountry", "Supplier ABNExempt" => "supplierABNExempt", "Supplier ABN" => "supplierABN", "Agency Branch" => "contactBranch", "Agency Divison" => "contactDivision", "Agency Postcode" => "contactPostcode", "" => "");
$headers; $headers;
$contractNoticeFields = array("importFile", "agencyName", "parentCN", "CNID", "publishDate", "amendDate", "contractStart", "contractEnd", "value", "description", "agencyID", "category", "procurementMethod", "atmID", "SONID", "confidentialityContract", "confidentialityContractReason", "confidentialityOutputs", "confidentialityOutputsReason", "consultancy", "consultancyReason", "amendmentReason", "supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry", "supplierABNExempt", "supplierABN", "contactBranch", "contactDivision", "contactPostcode"); $contractNoticeFields = array("importFile", "agencyName", "parentCN", "CNID", "publishDate", "amendDate", "contractStart", "contractEnd", "value", "description", "agencyID", "category", "procurementMethod", "atmID", "SONID", "confidentialityContract", "confidentialityContractReason", "confidentialityOutputs", "confidentialityOutputsReason", "consultancy", "consultancyReason", "amendmentReason", "supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry", "supplierABNExempt", "supplierABN", "contactBranch", "contactDivision", "contactPostcode");
$agencyFields = array("agencyName"); $agencyFields = array("agencyName");
$supplierFields = array("supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry", "supplierABNExempt", "supplierABN"); $supplierFields = array("supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry", "supplierABNExempt", "supplierABN");
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( '; $contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
foreach ($contractNoticeFields as $key => $f) { foreach ($contractNoticeFields as $key => $f) {
$contractNoticeInsertQ .= ($key == 0 ? "" : ", ")."?"; $contractNoticeInsertQ .= ($key == 0 ? "" : ", ")."?";
} }
$contractNoticeInsertQ .= ");"; $contractNoticeInsertQ .= ");";
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
} else if ($tablename == "supplierdetails") { } else if ($tablename == "supplierdetails") {
$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( '; $supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
foreach ($supplierFields as $key => $f) { foreach ($supplierFields as $key => $f) {
$supplierInsertQ .= ($key == 0 ? "" : ", ")."?"; $supplierInsertQ .= ($key == 0 ? "" : ", ")."?";
} }
$supplierInsertQ .= ");"; $supplierInsertQ .= ");";
$supplierInsertQ = $conn->prepare($supplierInsertQ); $supplierInsertQ = $conn->prepare($supplierInsertQ);
} else if ($tablename == "agency") { } else if ($tablename == "agency") {
$agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( '; $agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
foreach ($agencyFields as $key => $f) { foreach ($agencyFields as $key => $f) {
$agencyInsertQ .= ($key == 0 ? "" : ", ")."?"; $agencyInsertQ .= ($key == 0 ? "" : ", ")."?";
} }
$agencyInsertQ .= ");"; $agencyInsertQ .= ");";
$agencyInsertQ = $conn->prepare($agencyInsertQ); $agencyInsertQ = $conn->prepare($agencyInsertQ);
} }
   
while (($data = fgetcsv($handle, 1000, "\t")) !== false) { while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data); $num = count($data);
if ($row == 3) { if ($row == 3) {
$headers = $data; $headers = $data;
} elseif ($row > 3) { } elseif ($row > 3) {
if ($num > count($datamapping0711)) { if ($num > count($datamapping0711)) {
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data)); die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
} }
$contractNoticeInsert = Array(); $contractNoticeInsert = Array();
$supplierInsert = Array(); $supplierInsert = Array();
$agencyInsert = Array(); $agencyInsert = Array();
$contractNoticeInsert[] = $fname; $contractNoticeInsert[] = $fname;
$keys = array_keys($datamapping0711); $keys = array_keys($datamapping0711);
for ($c = 0; $c < $num; $c++) { for ($c = 0; $c < $num; $c++) {
$data[$c] = trim($data[$c], "="); $data[$c] = trim($data[$c], "=");
$data[$c] = trim($data[$c], "\""); $data[$c] = trim($data[$c], "\"");
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) { if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) {
if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") { if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
$data[$c] = str_replace("-A","00",$data[$c]); // make amendments really big numbers $data[$c] = str_replace("-A","00",$data[$c]); // make amendments really big numbers
  if (!is_numeric($data[$c])) die ($data[$c]." is not numeric");
if ($data[$c] > 0) { if ($data[$c] > 0) {
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} else { } else {
$contractNoticeInsert[] = 0; $contractNoticeInsert[] = 0;
} }
} elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") { } elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c])); $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
} else { } else {
if (strstr("\" =", $data[$c] > 0)) { if (strstr("\" =", $data[$c] > 0)) {
die("Invalid Description field" . $contractNoticeInsert); die("Invalid Description field" . $contractNoticeInsert);
} }
$contractNoticeInsert[] = $data[$c]; $contractNoticeInsert[] = $data[$c];
} }
} }
} else if ($tablename == "supplierdetails") { } else if ($tablename == "supplierdetails") {
if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) { if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) {
$supplierInsert[] = $data[$c]; $supplierInsert[] = $data[$c];
} }
} else if ($tablename == "agency") { } else if ($tablename == "agency") {
if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) { if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) {
$agencyInsert[] = $data[$c]; $agencyInsert[] = $data[$c];
} }
} }
} }
flush(); flush();
if ($tablename == "contractnotice") { if ($tablename == "contractnotice") {
$contractNoticeInsertQ->execute($contractNoticeInsert); $contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
$success++; $success++;
else else
echo $data[2] . " failed CN insert.<br>" . print_r($errors,true) echo $data[2] . " failed CN insert.<br>" . print_r($errors,true)
. " <br> ". print_r($contractNoticeInsert,true) ."<br> $row <br><br>\n"; . " <br> ". print_r($contractNoticeInsert,true) ."<br> $row <br><br>\n";
} else if ($tablename == "supplierdetails") { } else if ($tablename == "supplierdetails") {
$supplierInsertQ->execute($supplierInsert); $supplierInsertQ->execute($supplierInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
$success++; $success++;
else else
echo $data[2] . " failed agency insert.<br>" . print_r($errors,true) echo $data[2] . " failed agency insert.<br>" . print_r($errors,true)
. " <br> ". print_r($agencyInsert,true) ."<br> $row <br><br>\n"; . " <br> ". print_r($agencyInsert,true) ."<br> $row <br><br>\n";
   
} else if ($tablename == "agency") { } else if ($tablename == "agency") {
$agencyInsertQ->execute($agencyInsert); $agencyInsertQ->execute($agencyInsert);
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
$success++; $success++;
else else
echo $data[2] . " failed supplier insert.<br>" . print_r($errors,true) echo $data[2] . " failed supplier insert.<br>" . print_r($errors,true)
. " <br> ". print_r($supplierInsert,true) ."<br> $row <br><br>\n"; . " <br> ". print_r($supplierInsert,true) ."<br> $row <br><br>\n";
} }
flush(); flush();
//echo "<hr>\n"; //echo "<hr>\n";
} }
$row++; $row++;
} }
fclose($handle); fclose($handle);
return $success; return $success;
} }
   
$path = 'data/'; $path = 'data/';
if ($_REQUEST["fname"] == "") { if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>"; echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path); $dhandle = opendir($path);
// define an array to hold the files // define an array to hold the files
$files = array(); $files = array();
if ($dhandle) { if ($dhandle) {
// loop through all of the files // loop through all of the files
while (false !== ($fname = readdir($dhandle))) { while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) { if (($fname != '.') && ($fname != '..')) {
echo "<a href=\"