projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
apples.* fixes
apples.* fixes

file:a/admin/linkAmendments.php -> file:b/admin/linkAmendments.php
<?php <?php
include_once ("../lib/common.inc.php"); include_once ("../lib/common.inc.php");
$query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\''; $query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\'';
  $query = $conn->prepare($query);
  $query->execute();
  $query = 'update contractnotice set "supplierABN" = null where "supplierABN" = \'0\'';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE WHERE
c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
if ($row['parentCN'] != 0) { if ($row['parentCN'] != 0) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'); $row['parentCN'] . '\';');
echo 'UPDATE contractnotice SET "childCN" = \'' . echo 'UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' . $row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';'; $row['parentCN'] . '\';';
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $row['CNID'] . " linked to parent " . $row['parentCN'] . echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
} }
} }
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN
( (
SELECT "parentCN" SELECT "parentCN"
FROM contractnotice FROM contractnotice
GROUP BY "parentCN" GROUP BY "parentCN"
HAVING COUNT(*) > 1 HAVING COUNT(*) > 1
AND "parentCN" IS NOT NULL AND "parentCN" IS NOT NULL
) )
AND "childCN" IS NULL AND "childCN" IS NULL
GROUP BY "parentCN" having count(*) > 1'; GROUP BY "parentCN" having count(*) > 1';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg'])); $cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg']));
$last_cnid = array_pop($cnids); $last_cnid = array_pop($cnids);
foreach ($cnids as $cnid) { foreach ($cnids as $cnid) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' . $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$last_cnid . '\' where "CNID" = \'' . $last_cnid . '\' where "CNID" = \'' .
$cnid . '\';'); $cnid . '\';');
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[1] == 7 || $errors[1] ==0) if ($errors[1] == 7 || $errors[1] ==0)
echo $cnid . " linked to latest child " . $last_cnid . echo $cnid . " linked to latest child " . $last_cnid .
"<br>\n"; "<br>\n";
else print_r($errors); else print_r($errors);
} }
} }
?> ?>
   
file:b/admin/partialdata/scraper.txt (new)
  <?php
  date_default_timezone_set('Australia/Melbourne');
  $split = false;
  function format_bytes($size) {
  $units = array(' B', ' KB', ' MB', ' GB', ' TB');
  for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
  return round($size, 2).$units[$i];
  }
 
  $days = 4;
  if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
  $startDate = strtotime("05-Jun-2008");
  if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
 
  function getFile($startDate, $days, $minVal, $maxVal) {
  global $split;
  $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
  $file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
  echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
  $url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
  echo "<!-- $url -->";
  $current = file_get_contents($url);
  if (strpos($current,"There are no results that match your selection.")> 0 ) {
  echo "<font color=red>Empty file!</font><br>";
  }
  if (strpos($current,"Your search returned more than 1000 results.") === false) {
  file_put_contents($file, $current);
  echo "$file saved<br>";
  echo format_bytes(filesize($file))."<br>";
  echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
  echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
  echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
  flush();
  if (!isset($_REQUEST['split']) && !$split) {
  echo "Success so fetching next $days... <br>";
  getFile($endDate, $days, "" , "");
  }
  return true;
  } else {
  echo "<font color=red>Too many records!</font><br>";
  echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
  echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
  flush();
  if (!isset($_REQUEST['split']) && !$split) {
  echo "Failure so splitting ... <br>";
  doSplit($startDate, $days);
  }
  return false;
  }
  }
  function doSplit($startDate, $days) {
  global $split;
  $split = true;
  set_time_limit(20);
  getFile($startDate, $days, 0, 12000);
  getFile($startDate, $days, 12000, 16000);
  getFile($startDate, $days, 16000, 20000);
  getFile($startDate, $days, 20000, 30000);
  getFile($startDate, $days, 30000, 40000);
  // getFile($startDate, $days, 40000, 80000);
  getFile($startDate, $days, 40000, 60000);
  getFile($startDate, $days, 60000, 80000);
  // getFile($startDate, $days, 80000, 300000);
  getFile($startDate, $days, 80000, 150000);
  getFile($startDate, $days, 150000, 300000);
  getFile($startDate, $days, 300000, 999999999);
  }
  if (isset($_REQUEST['split'])) {
  doSplit($startDate, $days);
  } else {
  getFile($startDate, $days, "" , "");
  }
  ?>
 
file:a/displaySupplier.php -> file:b/displaySupplier.php
<?php <?php
   
include_once ("./lib/common.inc.php"); include_once ("./lib/common.inc.php");
if ($_REQUEST['supplier']) { if ($_REQUEST['supplier']) {
   
include_header("Supplier"); include_header("Supplier");
$supplierS = htmlentities(strip_tags($_REQUEST['supplier'])); $supplierS = htmlentities(strip_tags($_REQUEST['supplier']));
MethodCountGraph($supplierS); // MethodCountGraph($supplierS);
CnCGraph($supplierS); // CnCGraph($supplierS);
MethodValueGraph($supplierS); // MethodValueGraph($supplierS);
/* lobbyist ties /* lobbyist ties
   
links to ABR/ASIC/Google News/ASX/Court records links to ABR/ASIC/Google News/ASX/Court records
   
total value to various agencies (bar graph) total value to various agencies (bar graph)
   
spread procurement methods + percent consultancies + percent confidential (bar graph) spread procurement methods + percent consultancies + percent confidential (bar graph)
   
spread of contract values spread of contract values
   
spread of industries (textual?) */ spread of industries (textual?) */
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", $query = 'SELECT "CNID", "description", "value", "agencyName", "category",
"contractStart", "supplierName" "contractStart", "supplierName"
FROM contractnotice WHERE ' . FROM contractnotice WHERE ' .
$supplierQ . ' ' . $standardQ $supplierQ . ' ' . $standardQ
. ' ORDER BY value DESC'; . ' ORDER BY value DESC';
echo $query; echo $query;
$query = $conn->prepare($query); $query = $conn->prepare($query);
   
//$query->bindParam(":supplierName", $supplierName);  
  if ($supplierParts[0] > 0) {
$query->bindParam(":supplierABN", $supplierABN); $query->bindParam(":supplierABN", $supplierABN);
  } else {
  $query->bindParam(":supplierName", $supplierName);
  }
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">'; // echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">';
echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">'; // echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">';
   
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['value']), 2); $value = number_format(doubleval($row['value']), 2);
echo ("<tr> echo ("<tr>
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td>
<td><b>{$row['description']}</b></a></td> <td><b>{$row['description']}</b></a></td>
<td>\$$value</td><td>{$row['agencyName']}</td> <td>\$$value</td><td>{$row['agencyName']}</td>
<td>{$row['contractStart']}</td> <td>{$row['contractStart']}</td>
<td>{$row['supplierName']}</td> <td>{$row['supplierName']}</td>
</tr>"); </tr>");
} }
echo "</table>"; echo "</table>";
} else { } else {
/* /*
histograph of supplier size/value histograph of supplier size/value
*/ */
include_header("Suppliers"); include_header("Suppliers");
suppliersGraph(); suppliersGraph();
$query = 'SELECT SUM("value") as val, MAX("supplierName") as supplierName, "supplierABN",( $query = 'SELECT SUM("value") as val, MAX("supplierName") as supplierName, "supplierABN",(
case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
GROUP BY supplierID,"supplierABN" GROUP BY supplierID,"supplierABN"
ORDER BY val DESC ORDER BY val DESC
LIMIT 100'; LIMIT 100';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Position</th> <th>Position</th>
<th>Supplier</th> <th>Supplier</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
</tr> </tr>
</thead>"; </thead>";
$i = 1; $i = 1;
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['val']), 2); $value = number_format(doubleval($row['val']), 2);
$supplier = stripslashes($row['supplierABN'] . '-' . $row['suppliername']); $supplier = stripslashes($row['supplierABN'] . '-' . $row['suppliername']);
echo ("<tr><td>$i</td><td><b><a href=\"displaySupplier.php?supplier={$supplier}\">" . ucsmart($row['suppliername']) . "</a></b></td><td>\$$value</td></tr>\n"); echo ("<tr><td>$i</td><td><b><a href=\"displaySupplier.php?supplier={$supplier}\">" . ucsmart($row['suppliername']) . "</a></b></td><td>\$$value</td></tr>\n");
$i++; $i++;
} }
echo "</table>"; echo "</table>";
} }
include_footer(); include_footer();
?> ?>
   
file:a/futureindex.php (deleted)
<?php  
include_once ("./lib/common.inc.php");  
include_header("Home");  
echo ' <div class="hero-unit">  
<h1>Hello, world!</h1>  
<p>Vestibulum id ligula porta felis euismod semper. Integer posuere erat a ante venenatis dapibus posuere velit aliquet. Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec elit.</p>  
<p><a class="btn primary large">Learn more &raquo;</a></p>  
</div>';  
include_footer();  
?>  
 
 
file:a/lib/graphs.inc.php -> file:b/lib/graphs.inc.php
<?php <?php
$includedFlot = false; $includedFlot = false;
   
function includeFlot() { function includeFlot() {
if (!$includedFlot) { if (!$includedFlot) {
echo ' <!--[if lte IE 8]><script language="javascript" type="text/javascript" src="lib/flot/excanvas.min.js"></script><![endif]--> echo ' <!--[if lte IE 8]><script language="javascript" type="text/javascript" src="lib/flot/excanvas.min.js"></script><![endif]-->
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.stack.js"></script> <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.stack.js"></script>
   
'; ';
$includedFlot = true; $includedFlot = true;
} }
} }
   
function CNDistributionGraph() { function CNDistributionGraph() {
global $conn; global $conn;
includeFlot(); includeFlot();
?> ?>
<center><div id="cndist" style="width:900px;height:550px"></div></center> <center><div id="cndist" style="width:900px;height:550px"></div></center>
<script type="text/javascript"> <script type="text/javascript">
var placeholder = $("#cndist"); var placeholder = $("#cndist");
$(function () { $(function () {
   
var d1 = []; var d1 = [];
var d2 = []; var d2 = [];
<?php <?php
$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is null) as a group by cnid order by cnid'; $query = 'select cnid, count(*) from
  (select ("CNID"::integer - MOD("CNID"::integer,100)) as cnid
  from contractnotice where "CNID"::integer < 999999
  and "CNID" not like \'%-A%\'
  and "parentCN" is null) as a group by cnid order by cnid';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[2] != "") { if ($errors[2] != "") {
echo("Export terminated, db error" . print_r($errors, true)); echo("Export terminated, db error" . print_r($errors, true));
return Array(); return Array();
} }
   
foreach ($query->fetchAll() as $delta) { foreach ($query->fetchAll() as $delta) {
   
echo "d1.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n"; echo "d1.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n";
}; };
$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid'; $query = 'select cnid, count(*) from (select ("CNID"::integer - MOD("CNID"::integer,100)) as cnid
  from contractnotice where "CNID" not like \'%-A%\' and "parentCN" is not null)
  as a group by cnid order by cnid';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[2] != "") { if ($errors[2] != "") {
echo("Export terminated, db error" . print_r($errors, true)); echo("Export terminated, db error" . print_r($errors, true));
return Array(); return Array();
} }
   
foreach ($query->fetchAll() as $delta) { foreach ($query->fetchAll() as $delta) {
   
echo "d2.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n"; echo "d2.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n";
}; };
?> ?>
   
var data = [ var data = [
{ {
data: d1, data: d1,
series: { series: {
lines: { show: true }, lines: { show: true },
points: { show: true } points: { show: true }
}, },
bars: { show: true } bars: { show: true }
}, },
{ {
data: d2, data: d2,
series: { series: {