|
|
Alexander Sadleir
|
1 |
<?php |
|
|
Maxious
|
2 |
if (php_sapi_name() == "cli") { |
|
|
Alexander Sadleir
|
3 |
date_default_timezone_set('Australia/Melbourne'); |
|
|
|
4 |
$split = false; |
|
|
|
5 |
function format_bytes($size) { |
|
|
|
6 |
$units = array(' B', ' KB', ' MB', ' GB', ' TB'); |
|
|
|
7 |
for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024; |
|
|
|
8 |
return round($size, 2).$units[$i]; |
|
|
|
9 |
} |
|
|
|
10 |
|
|
|
|
11 |
$days = 4; |
|
|
|
12 |
if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; |
|
|
|
13 |
$startDate = strtotime("05-Jun-2008"); |
|
|
|
14 |
if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; |
|
|
|
15 |
|
|
|
|
16 |
function getFile($startDate, $days, $minVal, $maxVal) { |
|
|
|
17 |
global $split; |
|
|
|
18 |
$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); |
|
|
|
19 |
$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; |
|
|
|
20 |
echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; |
|
|
|
21 |
$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; |
|
|
|
22 |
echo "<!-- $url -->"; |
|
|
|
23 |
$current = file_get_contents($url); |
|
|
|
24 |
if (strpos($current,"There are no results that match your selection.")> 0 ) { |
|
|
|
25 |
echo "<font color=red>Empty file!</font><br>"; |
|
|
|
26 |
} |
|
|
|
27 |
if (strpos($current,"Your search returned more than 1000 results.") === false) { |
|
|
|
28 |
file_put_contents($file, $current); |
|
|
|
29 |
echo "$file saved<br>"; |
|
|
|
30 |
echo format_bytes(filesize($file))."<br>"; |
|
|
|
31 |
echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>'; |
|
|
|
32 |
echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>'; |
|
|
|
33 |
echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>'; |
|
|
|
34 |
flush(); |
|
|
|
35 |
if (!isset($_REQUEST['split']) && !$split) { |
|
|
|
36 |
echo "Success so fetching next $days... <br>"; |
|
|
|
37 |
getFile($endDate, $days, "" , ""); |
|
|
|
38 |
} |
|
|
|
39 |
return true; |
|
|
|
40 |
} else { |
|
|
|
41 |
echo "<font color=red>Too many records!</font><br>"; |
|
|
|
42 |
echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>'; |
|
|
|
43 |
echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>'; |
|
|
|
44 |
flush(); |
|
|
|
45 |
if (!isset($_REQUEST['split']) && !$split) { |
|
|
|
46 |
echo "Failure so splitting ... <br>"; |
|
|
|
47 |
doSplit($startDate, $days); |
|
|
|
48 |
} |
|
|
|
49 |
return false; |
|
|
|
50 |
} |
|
|
|
51 |
} |
|
|
|
52 |
function doSplit($startDate, $days) { |
|
|
|
53 |
global $split; |
|
|
|
54 |
$split = true; |
|
|
|
55 |
set_time_limit(20); |
|
|
|
56 |
getFile($startDate, $days, 0, 12000); |
|
|
|
57 |
getFile($startDate, $days, 12000, 16000); |
|
|
|
58 |
getFile($startDate, $days, 16000, 20000); |
|
|
|
59 |
getFile($startDate, $days, 20000, 30000); |
|
|
|
60 |
getFile($startDate, $days, 30000, 40000); |
|
|
|
61 |
// getFile($startDate, $days, 40000, 80000); |
|
|
|
62 |
getFile($startDate, $days, 40000, 60000); |
|
|
|
63 |
getFile($startDate, $days, 60000, 80000); |
|
|
|
64 |
// getFile($startDate, $days, 80000, 300000); |
|
|
|
65 |
getFile($startDate, $days, 80000, 150000); |
|
|
|
66 |
getFile($startDate, $days, 150000, 300000); |
|
|
|
67 |
getFile($startDate, $days, 300000, 999999999); |
|
|
|
68 |
} |
|
|
|
69 |
if (isset($_REQUEST['split'])) { |
|
|
|
70 |
doSplit($startDate, $days); |
|
|
|
71 |
} else { |
|
|
|
72 |
getFile($startDate, $days, "" , ""); |
|
|
|
73 |
} |
|
|
Maxious
|
74 |
} |
|
|
Alexander Sadleir
|
75 |
?> |
|
|
|
76 |
|