updated update script and merged lobbylens admin scripts
updated update script and merged lobbylens admin scripts

<?php <?php
$ch = curl_init(); $ch = curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_REFERER, "http://contractdashboard.lambdacomplex.org"); curl_setopt($ch, CURLOPT_REFERER, "http://contractdashboard.lambdacomplex.org");
$guid = "5f2f943e-15f4-4782-8fad-9a0fe83a2f47"; $guid = "5f2f943e-15f4-4782-8fad-9a0fe83a2f47";
$orgname = "Department of Foreign Affairs and Trade"; $orgname = "Department of Foreign Affairs and Trade";
   
//foreach $agency //foreach $agency
$url = "http://abr.business.gov.au/ABRXMLSearchRPC/ABRXMLSearch.asmx/ABRSearchByNameSimpleProtocol?name=".urlencode($orgname)."&postcode=&legalName=Y&tradingName=Y&NSW=Y&SA=Y&ACT=Y&VIC=Y&WA=Y&NT=Y&QLD=Y&TAS=Y&authenticationGuid=$guid"; $url = "http://abr.business.gov.au/ABRXMLSearchRPC/ABRXMLSearch.asmx/ABRSearchByNameSimpleProtocol?name=".urlencode($orgname)."&postcode=&legalName=Y&tradingName=Y&NSW=Y&SA=Y&ACT=Y&VIC=Y&WA=Y&NT=Y&QLD=Y&TAS=Y&authenticationGuid=$guid";
curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_URL, $url);
$body = curl_exec($ch); $body = curl_exec($ch);
$xml = new SimpleXMLElement($body); $xml = new SimpleXMLElement($body);
echo "<pre>"; echo "<pre>";
$result = $xml->response->searchResultsList->searchResultsRecord[0]; $result = $xml->response->searchResultsList->searchResultsRecord[0];
var_dump($result); var_dump($result);
echo "</pre>"; echo "</pre>";
echo "<b> Organisation:</b>" . $result->mainName->organisationName; echo "<b> Organisation:</b>" . $result->mainName->organisationName;
echo "<b> ABN:</b>".$result->ABN->identifierValue; echo "<b> ABN:</b>".$result->ABN->identifierValue;
// fi // fi
   
curl_close($ch); curl_close($ch);
   
?> ?>
  <?php
 
  // Returns portfolio scraped live from directory.gov.au
  // or null if can't find a portfolio
  function agency2portfolio ($agency) {
  static $cache = array();
  if (isset($cache[$agency])) { return $cache[$agency]; }
  $c = curl_init('http://www.directory.gov.au/searchres.php');
  curl_setopt($c, CURLOPT_POST, true);
  curl_setopt($c, CURLOPT_HEADER, false);
  curl_setopt($c, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($c, CURLOPT_REFERER, 'http://www.directory.gov.au/adsearch.php');
  curl_setopt($c, CURLOPT_USERAGENT, 'Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3');
  curl_setopt($c, CURLOPT_POSTFIELDS, array(
  'advkeywordfield' => '',
  'advorgunitfield' => $agency,
  'advrolefield' => '',
  'advsection' => 'All',
  'advsurnamefield' => '',
  'search' => 'Submit Query'
  ));
  $results = curl_exec($c);
 
  if (preg_match('#<span\s+class="standardlinks"><a\s+href="([^"]+)">#smi', $results, $m)) {
  $nextURL = $m[1];
  } else {
  $cache[$agency] = false; return false;
  }
 
  curl_setopt($c, CURLOPT_URL, 'http://www.directory.gov.au' . $nextURL);
  curl_setopt($c, CURLOPT_HTTPGET, true);
  curl_setopt($c, CURLOPT_REFERER, 'http://www.directory.gov.au/searchres.php');
  $results = curl_exec($c);
  if (preg_match('#portfolios:\s+([^<]+)#ims', $results, $m)) {
  $cache[$agency] = $m[1]; return $m[1];
  } else {
  $cache[$agency] = false; return false;
  }
  }
 
  ?>
 
<?php <?php
   
include_once("../common.inc.php"); include_once("../common.inc.php");
$year = $_REQUEST['year']; $year = $_REQUEST['year'];
$query = "SELECT supplierCountry, sum(value) FROM `contractnotice` where childCN = 0 AND (YEAR(contractStart) = $year) AND supplierCountry != 'Australia' GROUP BY supplierCountry "; $query = "SELECT supplierCountry, sum(value) FROM `contractnotice` where childCN = 0 AND (YEAR(contractStart) = $year) AND supplierCountry != 'Australia' GROUP BY supplierCountry ";
   
$result = mysql_query($query); $result = mysql_query($query);
if (!$result) echo mysql_error().$query; if (!$result) echo mysql_error().$query;
$countryValues = Array(); $countryValues = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
$supplierCountry = strtoupper($row['supplierCountry']); $supplierCountry = strtoupper($row['supplierCountry']);
if ($supplierCountry == "MYANMAR") $supplierCountry = "BURMA"; if ($supplierCountry == "MYANMAR") $supplierCountry = "BURMA";
if ($supplierCountry == "RUSSIAN FEDERATION") $supplierCountry = "RUSSIA"; if ($supplierCountry == "RUSSIAN FEDERATION") $supplierCountry = "RUSSIA";
if ($supplierCountry == "TAIWAN, PROVINCE OF CHINA") $supplierCountry = "TAIWAN"; if ($supplierCountry == "TAIWAN, PROVINCE OF CHINA") $supplierCountry = "TAIWAN";
$countryValues[$supplierCountry] = $row['sum(value)']; $countryValues[$supplierCountry] = $row['sum(value)'];
} }
mysql_free_result($result); mysql_free_result($result);
   
$query = "SELECT supplierPostcode, sum(value) FROM `contractnotice` where childCN = 0 AND (YEAR(contractStart) = $year) AND supplierCountry = 'Australia' GROUP BY supplierPostcode "; $query = "SELECT supplierPostcode, sum(value) FROM `contractnotice` where childCN = 0 AND (YEAR(contractStart) = $year) AND supplierCountry = 'Australia' GROUP BY supplierPostcode ";
   
$result = mysql_query($query); $result = mysql_query($query);
if (!$result) echo mysql_error().$query; if (!$result) echo mysql_error().$query;
$postcodeValues = Array(); $postcodeValues = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
$postcodeValues[$row['supplierPostcode']] = $row['sum(value)']; $postcodeValues[$row['supplierPostcode']] = $row['sum(value)'];
} }
mysql_free_result($result); mysql_free_result($result);
   
   
@mysql_select_db("tme") or die( "Unable to select database tme"); @mysql_select_db("tme") or die( "Unable to select database tme");
$query = "SELECT un, name $query = "SELECT un, name
FROM `country_simpl`"; FROM `country_simpl`";
   
$result = mysql_query($query); $result = mysql_query($query);
if (!$result) echo mysql_error().$query; if (!$result) echo mysql_error().$query;
$countryUN = Array(); $countryUN = Array();
while ($row = mysql_fetch_array($result, MYSQL_BOTH)) { while ($row = mysql_fetch_array($result, MYSQL_BOTH)) {
$countryUN[strtoupper($row['name'])] = $row['un']; $countryUN[strtoupper($row['name'])] = $row['un'];
} }
mysql_free_result($result); mysql_free_result($result);
   
echo "<table>"; echo "<table>";
   
// drop from table existing values // drop from table existing values
//$result = mysql_query('insert into indicator_values ( "variable", "area", "year", "value" ) VALUES (2107, '.$countryUN[strtoupper($countryName)].', '.$year.','.$countryValue.');'); //$result = mysql_query('insert into indicator_values ( "variable", "area", "year", "value" ) VALUES (2107, '.$countryUN[strtoupper($countryName)].', '.$year.','.$countryValue.');');
//if (!$result) echo mysql_error().$query; //if (!$result) echo mysql_error().$query;
   
   
foreach ($countryValues as $countryName => $countryValue) { foreach ($countryValues as $countryName => $countryValue) {
echo "<tr><td>$countryName<td>".$countryUN[strtoupper($countryName)]."<td>$countryValue</tr>"; echo "<tr><td>$countryName<td>".$countryUN[strtoupper($countryName)]."<td>$countryValue</tr>";
$query = 'insert into indicator_values ( `variable`, `area`, `year`, `value` ) VALUES (\'2107\', \''.$countryUN[strtoupper($countryName)].'\', \''.$year.'\',\''.$countryValue.'\');'; $query = 'insert into indicator_values ( `variable`, `area`, `year`, `value` ) VALUES (\'2107\', \''.$countryUN[strtoupper($countryName)].'\', \''.$year.'\',\''.$countryValue.'\');';
$result = mysql_query($query); $result = mysql_query($query);
if (!$result) echo mysql_error(); if (!$result) echo mysql_error();
} }
   
foreach ($postcodeValues as $postcode => $postcodeValue) { foreach ($postcodeValues as $postcode => $postcodeValue) {
echo "<tr><td>Australia<td>$postcode<td>$postcodeValue</tr>"; echo "<tr><td>Australia<td>$postcode<td>$postcodeValue</tr>";
$query = 'INSERT into indicator_values ( `variable`, `area`, `year`, `value` ) VALUES (\'2107\', \''.($postcode*1000).'\', \''.$year.'\',\''.$countryValue.'\');'; $query = 'INSERT into indicator_values ( `variable`, `area`, `year`, `value` ) VALUES (\'2107\', \''.($postcode*1000).'\', \''.$year.'\',\''.$countryValue.'\');';
$result = mysql_query($query); $result = mysql_query($query);
if (!$result) echo mysql_error(); if (!$result) echo mysql_error();
} }
   
echo "</table>"; echo "</table>";
?> ?>
  SELECT representative_id,firstname,surname,party,house,division_id, portfolio
  FROM portfolio2representative
  INNER JOIN representatives
  ON portfolio2representative.representative_id=representatives.id
  INNER JOIN portfolios
  ON portfolio2representative.portfolio_id=portfolios.id
 
  <?php
  error_reporting(E_ALL);
 
  $link = mysql_connect('localhost', 'team7', 'wlUsnD2xu9');
  if (!$link) {
  die('Could not connect: ' . mysql_error());
  }
  @mysql_select_db("team7") or die("Unable to select database");
 
  $unspscresult = mysql_query ("select * from UNSPSCcategories;");
  while ($row = mysql_fetch_assoc($unspscresult)) {
  $unspsc[$row['UNSPSC']] = $row['Title'];
  }
  $catsresult = mysql_query ("SELECT LEFT( categoryUNSPSC, 2 ) as cat , SUM( value ) as value
  FROM `contractnotice`
  GROUP BY cat ;");
  echo "<table>";
  while ($row = mysql_fetch_assoc($catsresult)) {
  $catName = $unspsc[$row['cat']."000000"].$row['cat'];
  if ($row['cat'] = "") $catName = "null";
 
  echo "<tr><td>$catName</td><td>".$row['value']."</td></tr>";
  }
  ?>
 
<?php <?php
$ch = curl_init(); $ch = curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_REFERER, "http://contractdashboard.lambdacomplex.org"); curl_setopt($ch, CURLOPT_REFERER, "http://contractdashboard.lambdacomplex.org");
   
$agency = "Department of Foreign Affairs and Trade"; $agency = "Department of Foreign Affairs and Trade";
   
//foreach $agency //foreach $agency
$url = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&key=ABQIAAAA95XYXN0cki3Yj_Sb71CFvBSgIPe2k9-DRgEVQvbvQDV8xmTiXRTBpHxrD7bSh5rl2lswY56769CyQQ&rsz=small&filter=1&gl=au&q=".urlencode($agency)."%20ABN%20site:.gov.au"; $url = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&key=ABQIAAAA95XYXN0cki3Yj_Sb71CFvBSgIPe2k9-DRgEVQvbvQDV8xmTiXRTBpHxrD7bSh5rl2lswY56769CyQQ&rsz=small&filter=1&gl=au&q=".urlencode($agency)."%20ABN%20site:.gov.au";
curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_URL, $url);
$body = curl_exec($ch); $body = curl_exec($ch);
$json = json_decode($body); $json = json_decode($body);
echo "<pre>"; echo "<pre>";
$result = $json->responseData->results[0]; $result = $json->responseData->results[0];
print_r($result); print_r($result);
echo "</pre>"; echo "</pre>";
echo "<b> Agency:</b> $agency"; echo "<b> Agency:</b> $agency";
echo "<b> URL:</b> http://".$result->visibleUrl; echo "<b> URL:</b> http://".$result->visibleUrl;
preg_match('/\d{2} \d{3} \d{3} \d{3}/i', $result->content,$abn); preg_match('/\d{2} \d{3} \d{3} \d{3}/i', $result->content,$abn);
echo "<b> ABN:</b> {$abn[0]}"; echo "<b> ABN:</b> {$abn[0]}";
// fi // fi
   
curl_close($ch); curl_close($ch);
   
?> ?>
<
<?php <?php
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
$path = '../data/'; $path = '../data/';
if ($_REQUEST["fname"] == "") { if ($_REQUEST["fname"] == "") {
echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>"; echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
$dhandle = opendir($path); $dhandle = opendir($path);
// define an array to hold the files // define an array to hold the files
$files = array(); $files = array();
   
if ($dhandle) { if ($dhandle) {
// loop through all of the files // loop through all of the files
while (false !== ($fname = readdir($dhandle))) { while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) { if (($fname != '.') && ($fname != '..')) {
echo "<a href=\"import.php?fname=$fname\">$fname</a>".filesize($path.$fname)."<br/>"; echo "<a href=\"import.php?fname=$fname\">$fname</a>".filesize($path.$fname)."<br/>";
} }
} }
} }
   
} else { } else {
$success =0; $success =0;
$fname = $_REQUEST["fname"]; $fname = $_REQUEST["fname"];
echo " ============== $fname ============== <br>"; echo " ============== $fname ============== <br>";
$row = 1; $row = 1;
$handle = fopen($path.$fname, "r"); $handle = fopen($path.$fname, "r");
//"t" mode string translates windows line breaks to unix //"t" mode string translates windows line breaks to unix
$datamapping0709 = Array("Agency" => "agencyName", "Parent CN ID" => "parentCN", "CN ID" => "CNID", "Publish Date" => "publishDate", "Amendment Date" => "amendDate", "Status" => "", "StartDate" => "contractStart" , "EndDate" => "contractEnd", "Value" => "value", "Description" => "description", "Agency Ref Id" => "agencyID", "Category" => "category", "Procurement Method" => "procurementMethod", "ATM ID" => "atmID", "SON ID" => "SONID","Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Consultancy" => "consultancy", "Consultancy Reason(s)" => "consultancyReason", "Amendment Reason" => "amendmentReason", "Supplier Name" => "supplierName", "Supplier Address" => "supplierAddress", "Supplier City" => "supplierCity", "Supplier Postcode" => "supplierPostcode", "Supplier Country" => "supplierCountry" , "Supplier ABNExempt" => "supplierABNExempt", "Supplier ABN" => "supplierABN" , "Agency Branch" => "contactBranch","Agency Divison" => "contactDivision", "Agency Postcode" => "contactPostcode" , "" => ""); $datamapping0709 = Array("Agency" => "agencyName", "Parent CN ID" => "parentCN", "CN ID" => "CNID", "Publish Date" => "publishDate", "Amendment Date" => "amendDate", "Status" => "", "StartDate" => "contractStart" , "EndDate" => "contractEnd", "Value" => "value", "Description" => "description", "Agency Ref Id" => "agencyID", "Category" => "category", "Procurement Method" => "procurementMethod", "ATM ID" => "atmID", "SON ID" => "SONID","Confidentiality - Contract" => "confidentialityContract", "Confidentiality - Contract Reason(s)" => "confidentialityContractReason", "Confidentiality - Outputs" => "confidentialityOutputs", "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", "Consultancy" => "consultancy", "Consultancy Reason(s)" => "consultancyReason", "Amendment Reason" => "amendmentReason", "Supplier Name" => "supplierName", "Supplier Address" => "supplierAddress", "Supplier City" => "supplierCity", "Supplier Postcode" => "supplierPostcode", "Supplier Country" => "supplierCountry" , "Supplier ABNExempt" => "supplierABNExempt", "Supplier ABN" => "supplierABN" , "Agency Branch" => "contactBranch","Agency Divison" => "contactDivision", "Agency Postcode" => "contactPostcode" , "" => "");
$headers; $headers;
$contractNoticeFields = Array("agencyName", "parentCN", "CNID", "publishDate", "amendDate", "contractStart" , "contractEnd", "value", "description", "agencyID", "category", "procurementMethod", "atmID", "SONID", "confidentialityContract", "confidentialityContractReason", "confidentialityOutputs", "confidentialityOutputsReason", "consultancy", "consultancyReason", "amendmentReason", "supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry" , "supplierABNExempt", "supplierABN", "contactBranch","contactDivision","contactPostcode" ); $contractNoticeFields = Array("agencyName", "parentCN", "CNID", "publishDate", "amendDate", "contractStart" , "contractEnd", "value", "description", "agencyID", "category", "procurementMethod", "atmID", "SONID", "confidentialityContract", "confidentialityContractReason", "confidentialityOutputs", "confidentialityOutputsReason", "consultancy", "consultancyReason", "amendmentReason", "supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry" , "supplierABNExempt", "supplierABN", "contactBranch","contactDivision","contactPostcode" );
$agencyFields = Array("agencyName"); $agencyFields = Array("agencyName");
$supplierFields = Array("supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry" , "supplierABNExempt", "supplierABN" ); $supplierFields = Array("supplierName", "supplierAddress", "supplierCity", "supplierPostcode", "supplierCountry" , "supplierABNExempt", "supplierABN" );
   
while (($data = fgetcsv($handle, 1000, "\t")) !== FALSE) { while (($data = fgetcsv($handle, 1000, "\t")) !== FALSE) {
$num = count($data); $num = count($data);
   
if ($row == 3) if ($row == 3)
{ {
$headers = $data; $headers = $data;
} }
else if ($row > 3) else if ($row > 3)
{ {
if ($num > count($datamapping0709)) if ($num > count($datamapping0709))
{ {
die ("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>".$fname.print_r($data)); die ("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>".$fname.print_r($data));
} }
$firstCN= false; $firstCN= false;
$firstSupplier= true; $firstSupplier= true;
$firstAgency= true; $firstAgency= true;
$contractNoticeInsert = "INSERT INTO contractnotice (importFile, ".implode(" , ",$contractNoticeFields). ") VALUES ( \"$fname\" "; $contractNoticeInsert = "INSERT INTO contractnotice (importFile, ".implode(" , ",$contractNoticeFields). ") VALUES ( \"$fname\" ";
$supplierInsert = "INSERT INTO supplierdetails (".implode(" , ",$supplierFields). ") VALUES ("; $supplierInsert = "INSERT INTO supplierdetails (".implode(" , ",$supplierFields). ") VALUES (";
$agencyInsert = "INSERT INTO agency (".implode(" , ",$agencyFields). ") VALUES ("; $agencyInsert = "INSERT INTO agency (".implode(" , ",$agencyFields). ") VALUES (";
$keys = array_keys($datamapping0709); $keys = array_keys($datamapping0709);
for ($c=0; $c < $num; $c++) { for ($c=0; $c < $num; $c++) {
$data[$c] = trim($data[$c],"="); $data[$c] = trim($data[$c],"=");
$data[$c] = trim($data[$c], "\""); $data[$c] = trim($data[$c], "\"");
if (in_array(($datamapping0709[$headers[$c]]),$contractNoticeFields)) if (in_array(($datamapping0709[$headers[$c]]),$contractNoticeFields))
{ {
if (($datamapping0709[$headers[$c]]) == "parentCN" || ($datamapping0709[$headers[$c]]) == "CNID") if (($datamapping0709[$headers[$c]]) == "parentCN" || ($datamapping0709[$headers[$c]]) == "CNID")
{ {
$data[$c] = substr($data[$c],2); $data[$c] = substr($data[$c],2);
if ($data[$c] >0) if ($data[$c] >0)
{ {
$contractNoticeInsert .= ($firstCN ? "" : ","). mysql_real_escape_string($data[$c]); $contractNoticeInsert .= ($firstCN ? "" : ","). mysql_real_escape_string($data[$c]);
} else } else
{ {
$contractNoticeInsert .= ($firstCN ? "" : ",")."''"; $contractNoticeInsert .= ($firstCN ? "" : ",")."''";
} }
} else if (($datamapping0709[$headers[$c]]) == "amendDate" || ($datamapping0709[$headers[$c]]) == "publishDate" || ($datamapping0709[$headers[$c]]) == "contractStart" || ($datamapping0709[$headers[$c]]) == "contractEnd") } else if (($datamapping0709[$headers[$c]]) == "amendDate" || ($datamapping0709[$headers[$c]]) == "publishDate" || ($datamapping0709[$headers[$c]]) == "contractStart" || ($datamapping0709[$headers[$c]]) == "contractEnd")
{ {
$contractNoticeInsert .= ($firstCN ? "" : ","). "'".date( 'Y-m-d H:i:s', strtotime($data[$c])). "'"; $contractNoticeInsert .= ($firstCN ? "" : ","). "'".date( 'Y-m-d H:i:s', strtotime($data[$c])). "'";
} else } else
{ {
if (strstr("\" =",$data[$c] > 0)) if (strstr("\" =",$data[$c] > 0))
{ {
die ("Invalid Description field" . $contractNoticeInsert); die ("Invalid Description field" . $contractNoticeInsert);
} }
$contractNoticeInsert .= ($firstCN ? "" : ",") . "'". mysql_real_escape_string($data[$c])."'"; $contractNoticeInsert .= ($firstCN ? "" : ",") . "'". mysql_real_escape_string($data[$c])."'";
} }
$firstCN = false; $firstCN = false;
} }
if (in_array(($datamapping0709[$headers[$c]]),$supplierFields)) if (in_array(($datamapping0709[$headers[$c]]),$supplierFields))
{ {
$supplierInsert .= ($firstSupplier ? "" : ",") . "'". mysql_real_escape_string($data[$c])."'"; $supplierInsert .= ($firstSupplier ? "" : ",") . "'". mysql_real_escape_string($data[$c])."'";
$firstSupplier = false; $firstSupplier = false;
} }
if (in_array(($datamapping0709[$headers[$c]]),$agencyFields)) if (in_array(($datamapping0709[$headers[$c]]),$agencyFields))
{ {