projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Improve OS validation
Improve OS validation

file:a/admin/fixoldamend.php -> file:b/admin/fixoldamend.php
<?php <?php
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
   
$query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\''; $query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\'';
$result = $conn->prepare($query); $result = $conn->prepare($query);
$result->execute(); $result->execute();
$query = 'update contractnotice set "childCN" = null where "childCN" = \'0\''; $query = 'update contractnotice set "childCN" = null where "childCN" = \'0\'';
$result = $conn->prepare($query); $result = $conn->prepare($query);
$result->execute(); $result->execute();
$query = 'select "CNID","parentCN" from contractnotice where "CNID" like \'%00_\' and "parentCN" is not null'; $query = 'select "CNID","parentCN" from contractnotice where "CNID" like \'%00_\' and "parentCN" is not null';
$result = $conn->prepare($query); $result = $conn->prepare($query);
$result->execute(); $result->execute();
foreach ($result->fetchAll() as $record) { foreach ($result->fetchAll() as $record) {
$oldCN = $record['CNID']; $oldCN = $record['CNID'];
$parentCN = substr($oldCN,0,-3); $parentCN = substr($oldCN,0,-3);
if ($parentCN == $record['parentCN']) { if ($parentCN == $record['parentCN']) {
$newCN = $parentCN . "-A". substr($oldCN,-1); $newCN = $parentCN . "-A". substr($oldCN,-1);
$updateresult = $conn->exec('UPDATE contractnotice SET "CNID" = $updateresult = $conn->exec('UPDATE contractnotice SET "CNID" =
\'' . $newCN . '\' where "CNID" = \'' . $oldCN . '\';'); \'' . $newCN . '\' where "CNID" = \'' . $oldCN . '\';');
echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n"; echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n";
} }
else { else {
echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n"; // echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n";
} }
} }
   
file:a/admin/linkAmendments.php -> file:b/admin/linkAmendments.php
<?php <?php
include_once ("../lib/common.inc.php"); include_once ("../lib/common.inc.php");
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE WHERE
c."parentCN" > 0 AND p."childCN" IS NULL '; c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'); $row['parentCN'] . '\';');
echo 'UPDATE contractnotice SET "childCN" = \'' . echo 'UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'; $row['parentCN'] . '\';';
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $row['CNID'] . " linked to parent " . $row['parentCN'] . echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
   
} }
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN
( (
SELECT "parentCN" SELECT "parentCN"
FROM contractnotice FROM contractnotice
GROUP BY "parentCN" GROUP BY "parentCN"
HAVING COUNT(*) > 1 HAVING COUNT(*) > 1
AND "parentCN" != 0 AND "parentCN" IS NOT NULL
) )
AND "childCN" = 0 AND "childCN" IS NULL
GROUP BY "parentCN" having count(*) > 1'; GROUP BY "parentCN" having count(*) > 1';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg'])); $cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg']));
$last_cnid = array_pop($cnids); $last_cnid = array_pop($cnids);
foreach ($cnids as $cnid) { foreach ($cnids as $cnid) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$last_cnid . '\' where "CNID" = \'' . $last_cnid . '\' where "CNID" = \'' .
$cnid . '\';'); $cnid . '\';');
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $cnid . " linked to latest child " . $last_cnid . echo $cnid . " linked to latest child " . $last_cnid .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
} }
} }
?> ?>
   
file:a/admin/partialdata/scraper.php (deleted)
<?php  
date_default_timezone_set('Australia/Melbourne');  
$split = false;  
function format_bytes($size) {  
$units = array(' B', ' KB', ' MB', ' GB', ' TB');  
for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;  
return round($size, 2).$units[$i];  
}  
 
$days = 4;  
if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];  
$startDate = strtotime("05-Jun-2008");  
if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];  
 
function getFile($startDate, $days, $minVal, $maxVal) {  
global $split;  
$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");  
$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';  
echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";  
$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";  
echo "<!-- $url -->";  
$current = file_get_contents($url);  
if (strpos($current,"There are no results that match your selection.")> 0 ) {  
echo "<font color=red>Empty file!</font><br>";  
}  
if (strpos($current,"Your search returned more than 1000 results.") === false) {  
file_put_contents($file, $current);  
echo "$file saved<br>";  
echo format_bytes(filesize($file))."<br>";  
echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';  
echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';  
echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';  
flush();  
if (!isset($_REQUEST['split']) && !$split) {  
echo "Success so fetching next $days... <br>";  
getFile($endDate, $days, "" , "");  
}  
return true;  
} else {  
echo "<font color=red>Too many records!</font><br>";  
echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';  
echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';  
flush();  
if (!isset($_REQUEST['split']) && !$split) {  
echo "Failure so splitting ... <br>";  
doSplit($startDate, $days);  
}  
return false;  
}  
}  
function doSplit($startDate, $days) {  
global $split;  
$split = true;  
set_time_limit(20);  
getFile($startDate, $days, 0, 12000);  
getFile($startDate, $days, 12000, 16000);  
getFile($startDate, $days, 16000, 20000);  
getFile($startDate, $days, 20000, 30000);  
getFile($startDate, $days, 30000, 40000);  
// getFile($startDate, $days, 40000, 80000);  
getFile($startDate, $days, 40000, 60000);  
getFile($startDate, $days, 60000, 80000);  
// getFile($startDate, $days, 80000, 300000);  
getFile($startDate, $days, 80000, 150000);  
getFile($startDate, $days, 150000, 300000);  
getFile($startDate, $days, 300000, 999999999);  
}  
if (isset($_REQUEST['split'])) {  
doSplit($startDate, $days);  
} else {  
getFile($startDate, $days, "" , "");  
}  
?>  
 
file:a/admin/updateUNSPSC.php -> file:b/admin/updateUNSPSC.php
<?php <?php
error_reporting(E_ALL); error_reporting(E_ALL);
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
$unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); $unspscresult = $conn->prepare('select * from "UNSPSCcategories";');
$unspscresult->execute(); $unspscresult->execute();
foreach ($unspscresult->fetchAll() as $row) { foreach ($unspscresult->