Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
Conflicts:
admin/updateUNSPSC.php
--- a/admin/fixoldamend.php
+++ b/admin/fixoldamend.php
@@ -20,7 +20,7 @@
echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n";
}
else {
- echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n";
+// echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n";
}
}
--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -2,7 +2,7 @@
include_once ("../lib/common.inc.php");
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE
-c."parentCN" > 0 AND p."childCN" IS NULL ';
+c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
$query = $conn->prepare($query);
$query->execute();
databaseError($conn->errorInfo());
@@ -28,9 +28,9 @@
FROM contractnotice
GROUP BY "parentCN"
HAVING COUNT(*) > 1
- AND "parentCN" != 0
+ AND "parentCN" IS NOT NULL
)
-AND "childCN" = 0
+AND "childCN" IS NULL
GROUP BY "parentCN" having count(*) > 1';
$query = $conn->prepare($query);
$query->execute();
--- a/admin/partialdata/scraper.php
+++ /dev/null
@@ -1,74 +1,1 @@
-<?php
-date_default_timezone_set('Australia/Melbourne');
-$split = false;
-function format_bytes($size) {
- $units = array(' B', ' KB', ' MB', ' GB', ' TB');
- for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
- return round($size, 2).$units[$i];
-}
-$days = 4;
-if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
-$startDate = strtotime("05-Jun-2008");
-if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
-
-function getFile($startDate, $days, $minVal, $maxVal) {
-global $split;
- $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
-$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
-echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
-$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
-echo "<!-- $url -->";
-$current = file_get_contents($url);
-if (strpos($current,"There are no results that match your selection.")> 0 ) {
- echo "<font color=red>Empty file!</font><br>";
-}
-if (strpos($current,"Your search returned more than 1000 results.") === false) {
- file_put_contents($file, $current);
- echo "$file saved<br>";
- echo format_bytes(filesize($file))."<br>";
- echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
- echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
- echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
- flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Success so fetching next $days... <br>";
-getFile($endDate, $days, "" , "");
-}
- return true;
-} else {
- echo "<font color=red>Too many records!</font><br>";
- echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
- echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
- flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Failure so splitting ... <br>";
- doSplit($startDate, $days);
-}
- return false;
-}
-}
-function doSplit($startDate, $days) {
-global $split;
-$split = true;
-set_time_limit(20);
-getFile($startDate, $days, 0, 12000);
-getFile($startDate, $days, 12000, 16000);
- getFile($startDate, $days, 16000, 20000);
- getFile($startDate, $days, 20000, 30000);
- getFile($startDate, $days, 30000, 40000);
-// getFile($startDate, $days, 40000, 80000);
- getFile($startDate, $days, 40000, 60000);
- getFile($startDate, $days, 60000, 80000);
-// getFile($startDate, $days, 80000, 300000);
- getFile($startDate, $days, 80000, 150000);
- getFile($startDate, $days, 150000, 300000);
- getFile($startDate, $days, 300000, 999999999);
-}
-if (isset($_REQUEST['split'])) {
- doSplit($startDate, $days);
-} else {
- getFile($startDate, $days, "" , "");
-}
-?>
-
--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -19,13 +19,26 @@
$unspsc[$armor] = $row['UNSPSC'];
$erre = str_replace("er", "re", $row['Title']);
$unspsc[$erre] = $row['UNSPSC'];
+ $center = str_replace("center", "centre", $row['Title']);
+ $unspsc[$center] = $row['UNSPSC'];
+ $accessory = str_replace("accesor", "accessor", $row['Title']);
+ $unspsc[$accessory] = $row['UNSPSC'];
+ $lyslyz = str_replace("lyz", "lys", $row['Title']);
+ $unspsc[$lyslyz] = $row['UNSPSC'];
$tire = str_replace("ire", "yre", $row['Title']);
$unspsc[$tire] = $row['UNSPSC'];
- $aeroplane = str_replace("airplane","aeroplane", $row['Title']);
- $unspsc[$aeroplane] = $row['UNSPSC'];
- $lyslyz = str_replace("lyz", "lys", $row['Title']);
- $unspsc[$lyslyz] = $row['UNSPSC'];
+
+ $pe = str_replace("pe", "pae", $row['Title']);
+ $unspsc[$pe] = $row['UNSPSC'];
+ $ane = str_replace("ane", "anae", $row['Title']);
+ $unspsc[$ane] = $row['UNSPSC'];
+ $airo = str_replace("airplane", "aeroplane", $row['Title']);
+ $unspsc[$airo] = $row['UNSPSC'];
// some divergence from standard
+ $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']);
+ $unspsc[$forensicit] = $row['UNSPSC'];
+ $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']);
+ $unspsc[$powercable] = $row['UNSPSC'];
$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);
$unspsc[$tobacco] = $row['UNSPSC'];
$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);