Merge branch 'master' of /git/contractdashboard
[contractdashboard.git] / admin / partialdata / scraper.txt
Alexander Sadleir 1 <?php
Maxious 2 date_default_timezone_set('Australia/Melbourne');
Alexander Sadleir 3 $split = false;
4 function format_bytes($size) {
5 $units = array(' B', ' KB', ' MB', ' GB', ' TB');
6 for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
7 return round($size, 2).$units[$i];
8 }
9
10 $days = 4;
11 if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
12 $startDate = strtotime("05-Jun-2008");
13 if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
14
15 function getFile($startDate, $days, $minVal, $maxVal) {
16 global $split;
17 $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
18 $file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
19 echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
20 $url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
21 echo "<!-- $url -->";
22 $current = file_get_contents($url);
23 if (strpos($current,"There are no results that match your selection.")> 0 ) {
24 echo "<font color=red>Empty file!</font><br>";
25 }
26 if (strpos($current,"Your search returned more than 1000 results.") === false) {
27 file_put_contents($file, $current);
28 echo "$file saved<br>";
29 echo format_bytes(filesize($file))."<br>";
30 echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
31 echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
32 echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
33 flush();
34 if (!isset($_REQUEST['split']) && !$split) {
35 echo "Success so fetching next $days... <br>";
36 getFile($endDate, $days, "" , "");
37 }
38 return true;
39 } else {
40 echo "<font color=red>Too many records!</font><br>";
41 echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
42 echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
43 flush();
44 if (!isset($_REQUEST['split']) && !$split) {
45 echo "Failure so splitting ... <br>";
46 doSplit($startDate, $days);
47 }
48 return false;
49 }
50 }
51 function doSplit($startDate, $days) {
52 global $split;
53 $split = true;
54 set_time_limit(20);
55 getFile($startDate, $days, 0, 12000);
56 getFile($startDate, $days, 12000, 16000);
57 getFile($startDate, $days, 16000, 20000);
58 getFile($startDate, $days, 20000, 30000);
59 getFile($startDate, $days, 30000, 40000);
60 // getFile($startDate, $days, 40000, 80000);
61 getFile($startDate, $days, 40000, 60000);
62 getFile($startDate, $days, 60000, 80000);
63 // getFile($startDate, $days, 80000, 300000);
64 getFile($startDate, $days, 80000, 150000);
65 getFile($startDate, $days, 150000, 300000);
66 getFile($startDate, $days, 300000, 999999999);
67 }
68 if (isset($_REQUEST['split'])) {
69 doSplit($startDate, $days);
70 } else {
71 getFile($startDate, $days, "" , "");
72 }
73 ?>
Maxious 74