| /admin/neo4jimporter/target/ | |
| /admin/neo4jimporter/.settings/ | |
| /nbproject/private/ |
| <?php | <?php |
| include_once("../lib/common.inc.php"); | include_once("../lib/common.inc.php"); |
| $query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\''; | $query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\''; |
| $result = $conn->prepare($query); | $result = $conn->prepare($query); |
| $result->execute(); | $result->execute(); |
| $query = 'update contractnotice set "childCN" = null where "childCN" = \'0\''; | $query = 'update contractnotice set "childCN" = null where "childCN" = \'0\''; |
| $result = $conn->prepare($query); | $result = $conn->prepare($query); |
| $result->execute(); | $result->execute(); |
| $query = 'select "CNID","parentCN" from contractnotice where "CNID" like \'%00_\' and "parentCN" is not null'; | $query = 'select "CNID","parentCN" from contractnotice where "CNID" like \'%00_\' and "parentCN" is not null'; |
| $result = $conn->prepare($query); | $result = $conn->prepare($query); |
| $result->execute(); | $result->execute(); |
| foreach ($result->fetchAll() as $record) { | foreach ($result->fetchAll() as $record) { |
| $oldCN = $record['CNID']; | $oldCN = $record['CNID']; |
| $parentCN = substr($oldCN,0,-3); | $parentCN = substr($oldCN,0,-3); |
| if ($parentCN == $record['parentCN']) { | if ($parentCN == $record['parentCN']) { |
| $newCN = $parentCN . "-A". substr($oldCN,-1); | $newCN = $parentCN . "-A". substr($oldCN,-1); |
| $updateresult = $conn->exec('UPDATE contractnotice SET "CNID" = | $updateresult = $conn->exec('UPDATE contractnotice SET "CNID" = |
| \'' . $newCN . '\' where "CNID" = \'' . $oldCN . '\';'); | \'' . $newCN . '\' where "CNID" = \'' . $oldCN . '\';'); |
| echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n"; | echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n"; |
| } | } |
| else { | else { |
| echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n"; | // echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n"; |
| } | } |
| } | } |
| <?php | |
| date_default_timezone_set('Australia/Melbourne'); | |
| $split = false; | |
| function format_bytes($size) { | |
| $units = array(' B', ' KB', ' MB', ' GB', ' TB'); | |
| for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024; | |
| return round($size, 2).$units[$i]; | |
| } | |
| $days = 4; | |
| if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; | |
| $startDate = strtotime("05-Jun-2008"); | |
| if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; | |
| function getFile($startDate, $days, $minVal, $maxVal) { | |
| global $split; | |
| $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); | |
| $file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; | |
| echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; | |
| $url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; | |
| echo "<!-- $url -->"; | |
| $current = file_get_contents($url); | |
| if (strpos($current,"There are no results that match your selection.")> 0 ) { | |
| echo "<font color=red>Empty file!</font><br>"; | |
| } | |
| if (strpos($current,"Your search returned more than 1000 results.") === false) { | |
| file_put_contents($file, $current); | |
| echo "$file saved<br>"; | |
| echo format_bytes(filesize($file))."<br>"; | |
| echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>'; | |
| echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>'; | |
| echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>'; | |
| flush(); | |
| if (!isset($_REQUEST['split']) && !$split) { | |
| echo "Success so fetching next $days... <br>"; | |
| getFile($endDate, $days, "" , ""); | |
| } | |
| return true; | |
| } else { | |
| echo "<font color=red>Too many records!</font><br>"; | |
| echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>'; | |
| echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>'; | |
| flush(); | |
| if (!isset($_REQUEST['split']) && !$split) { | |
| echo "Failure so splitting ... <br>"; | |
| doSplit($startDate, $days); | |
| } | |
| return false; | |
| } | |
| } | |
| function doSplit($startDate, $days) { | |
| global $split; | |
| $split = true; | |
| set_time_limit(20); | |
| getFile($startDate, $days, 0, 12000); | |
| getFile($startDate, $days, 12000, 16000); | |
| getFile($startDate, $days, 16000, 20000); | |
| getFile($startDate, $days, 20000, 30000); | |
| getFile($startDate, $days, 30000, 40000); | |
| // getFile($startDate, $days, 40000, 80000); | |
| getFile($startDate, $days, 40000, 60000); | |
| getFile($startDate, $days, 60000, 80000); | |
| // getFile($startDate, $days, 80000, 300000); | |
| getFile($startDate, $days, 80000, 150000); | |
| getFile($startDate, $days, 150000, 300000); | |
| getFile($startDate, $days, 300000, 999999999); | |
| } | |
| if (isset($_REQUEST['split'])) { | |
| doSplit($startDate, $days); | |
| } else { | |
| getFile($startDate, $days, "" , ""); | |
| } | |
| ?> | |
| <?php | <?php |
| error_reporting(E_ALL); | error_reporting(E_ALL); |
| include_once("../lib/common.inc.php"); | include_once("../lib/common.inc.php"); |
| $unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); | $unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); |
| $unspscresult->execute(); | $unspscresult->execute(); |
| foreach ($unspscresult->fetchAll() as $row) { | foreach ($unspscresult->fetchAll() as $row) { |
| $unspsc[$row['Title']] = $row['UNSPSC']; | $unspsc[$row['Title']] = $row['UNSPSC']; |
| // some Australian spellings | // some Australian spellings |
| $isiz = str_replace("iz", "is", $row['Title']); | $isiz = str_replace("iz", "is", $row['Title']); |
| $unspsc[$isiz] = $row['UNSPSC']; | $unspsc[$isiz] = $row['UNSPSC']; |
| $filfill = str_replace("fill", "fil", $row['Title']); | $filfill = str_replace("fill", "fil", $row['Title']); |
| $unspsc[$filfill] = $row['UNSPSC']; | $unspsc[$filfill] = $row['UNSPSC']; |
| $defence = str_replace("efense", "efence", $row['Title']); | $defence = str_replace("efense", "efence", $row['Title']); |
| $unspsc[$defence] = $row['UNSPSC']; | $unspsc[$defence] = $row['UNSPSC']; |
| $armor = str_replace("rmored", "rmoured", $row['Title']); | $armor = str_replace("rmored", "rmoured", $row['Title']); |
| $unspsc[$armor] = $row['UNSPSC']; | $unspsc[$armor] = $row['UNSPSC']; |
| $erre = str_replace("er", "re", $row['Title']); | $erre = str_replace("er", "re", $row['Title']); |
| $unspsc[$erre] = $row['UNSPSC']; | $unspsc[$erre] = $row['UNSPSC']; |
| $center = str_replace("center", "centre", $row['Title']); | |
| $unspsc[$center] = $row['UNSPSC']; | |
| $accessory = str_replace("accesor", "accessor", $row['Title']); | |
| $unspsc[$accessory] = $row['UNSPSC']; | |
| $lyslyz = str_replace("lyz", "lys", $row['Title']); | $lyslyz = str_replace("lyz", "lys", $row['Title']); |
| $unspsc[$lyslyz] = $row['UNSPSC']; | $unspsc[$lyslyz] = $row['UNSPSC']; |
| $tire = str_replace("ire", "yre", $row['Title']); | |
| $unspsc[$tire] = $row['UNSPSC']; | |
| $pe = str_replace("pe", "pae", $row['Title']); | |
| $unspsc[$pe] = $row['UNSPSC']; | |
| $ane = str_replace("ane", "anae", $row['Title']); | |
| $unspsc[$ane] = $row['UNSPSC']; | |
| $airo = str_replace("airplane", "aeroplane", $row['Title']); | |
| $unspsc[$airo] = $row['UNSPSC']; | |
| // some divergence from standard | // some divergence from standard |
| $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']); | |
| $unspsc[$forensicit] = $row['UNSPSC']; | |
| $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']); | |
| $unspsc[$powercable] = $row['UNSPSC']; | |
| $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); | $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); |
| $unspsc[$tobacco] = $row['UNSPSC']; | $unspsc[$tobacco] = $row['UNSPSC']; |
| $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); | $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); |
| $unspsc[$architect] = $row['UNSPSC']; | $unspsc[$architect] = $row['UNSPSC']; |
| // some just plain wrong | // some just plain wrong |
| $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); | $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); |
| $unspsc[$noOilRigs] = $row['UNSPSC']; | $unspsc[$noOilRigs] = $row['UNSPSC']; |
| } | } |
| $query = 'SELECT "CNID","category","value" | $query = 'SELECT "CNID","category","value" |
| FROM contractnotice | FROM contractnotice |
| WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; | WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; |