gitphp 0.2.9.1 :: contractdashboard.git/commitdiff

OpenSpending csv updates

7 files changed: (show all)
admin/linkAmendments.php
admin/partialdata/scraper.php (deleted)
admin/updateUNSPSC.php
australian_federal_government_contract_spending.json
displayBubbletree.php
exportData.csv.php
robots.txt (new)

file:a/admin/linkAmendments.php -> file:b/admin/linkAmendments.php

<?php	<?php
include_once ("../lib/common.inc.php");	include_once ("../lib/common.inc.php");
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"	$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE	WHERE
c."parentCN" > 0 AND p."childCN" IS NULL ';	c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
$query = $conn->prepare($query);	$query = $conn->prepare($query);
$query->execute();	$query->execute();
databaseError($conn->errorInfo());	databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) {	foreach ($query->fetchAll() as $row) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' .	$conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' .	$row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';');	$row['parentCN'] . '\';');
echo 'UPDATE contractnotice SET "childCN" = \'' .	echo 'UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' .	$row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';';	$row['parentCN'] . '\';';
$errors = $conn->errorInfo();	$errors = $conn->errorInfo();

if ($errors[1] == 7 \|\| $errors[1] ==0)	if ($errors[1] == 7 \|\| $errors[1] ==0)
echo $row['CNID'] . " linked to parent " . $row['parentCN'] .	echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
"<br>\n";	"<br>\n";
else print_r($errors);	else print_r($errors);

}	}
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0	// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN	$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN
(	(
SELECT "parentCN"	SELECT "parentCN"
FROM contractnotice	FROM contractnotice
GROUP BY "parentCN"	GROUP BY "parentCN"
HAVING COUNT(*) > 1	HAVING COUNT(*) > 1
AND "parentCN" != 0	AND "parentCN" IS NOT NULL
)	)
AND "childCN" = 0	AND "childCN" IS NULL
GROUP BY "parentCN" having count(*) > 1';	GROUP BY "parentCN" having count(*) > 1';
$query = $conn->prepare($query);	$query = $conn->prepare($query);
$query->execute();	$query->execute();
databaseError($conn->errorInfo());	databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) {	foreach ($query->fetchAll() as $row) {
$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg']));	$cnids = explode(",",str_replace(Array("{","}"),"",$row['array_agg']));
$last_cnid = array_pop($cnids);	$last_cnid = array_pop($cnids);
foreach ($cnids as $cnid) {	foreach ($cnids as $cnid) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' .	$conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$last_cnid . '\' where "CNID" = \'' .	$last_cnid . '\' where "CNID" = \'' .
$cnid . '\';');	$cnid . '\';');

$errors = $conn->errorInfo();	$errors = $conn->errorInfo();

if ($errors[1] == 7 \|\| $errors[1] ==0)	if ($errors[1] == 7 \|\| $errors[1] ==0)
echo $cnid . " linked to latest child " . $last_cnid .	echo $cnid . " linked to latest child " . $last_cnid .
"<br>\n";	"<br>\n";
else print_r($errors);	else print_r($errors);
}	}
}	}
?>	?>

file:a/admin/partialdata/scraper.php (deleted)

<?php
date_default_timezone_set('Australia/Melbourne');
$split = false;
function format_bytes($size) {
$units = array(' B', ' KB', ' MB', ' GB', ' TB');
for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
return round($size, 2).$units[$i];
}

$days = 4;
if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
$startDate = strtotime("05-Jun-2008");
if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];

function getFile($startDate, $days, $minVal, $maxVal) {
global $split;
$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
echo "<!-- $url -->";
$current = file_get_contents($url);
if (strpos($current,"There are no results that match your selection.")> 0 ) {
echo "<font color=red>Empty file!</font><br>";
}
if (strpos($current,"Your search returned more than 1000 results.") === false) {
file_put_contents($file, $current);
echo "$file saved<br>";
echo format_bytes(filesize($file))."<br>";
echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
echo '<a href="?startDate='.$endDate.'&days='.($days2).'">Load next '.($days2).' days </a><br>';
echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
flush();
if (!isset($_REQUEST['split']) && !$split) {
echo "Success so fetching next $days... <br>";
getFile($endDate, $days, "" , "");
}
return true;
} else {
echo "<font color=red>Too many records!</font><br>";
echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
flush();
if (!isset($_REQUEST['split']) && !$split) {
echo "Failure so splitting ... <br>";
doSplit($startDate, $days);
}
return false;
}
}
function doSplit($startDate, $days) {
global $split;
$split = true;
set_time_limit(20);
getFile($startDate, $days, 0, 12000);
getFile($startDate, $days, 12000, 16000);
getFile($startDate, $days, 16000, 20000);
getFile($startDate, $days, 20000, 30000);
getFile($startDate, $days, 30000, 40000);
// getFile($startDate, $days, 40000, 80000);
getFile($startDate, $days, 40000, 60000);
getFile($startDate, $days, 60000, 80000);
// getFile($startDate, $days, 80000, 300000);
getFile($startDate, $days, 80000, 150000);
getFile($startDate, $days, 150000, 300000);
getFile($startDate, $days, 300000, 999999999);
}
if (isset($_REQUEST['split'])) {
doSplit($startDate, $days);
} else {
getFile($startDate, $days, "" , "");
}
?>

file:a/admin/updateUNSPSC.php -> file:b/admin/updateUNSPSC.php

file:a/australian_federal_government_contract_spending.json -> file:b/australian_federal_government_contract_spending.json

{	{
"dataset":{	"dataset": {
"name":"australian_federal_government_contract_spending",	"languages": [
"label":"Australian Federal Government Contract Spending",	"en"
"description":"Spending by Australian Federal Government agencies on goods and services from 2007 onwards.",	],
"currency":"AUD",	"currency": "AUD",
"unique_keys":[	"name": "australian_federal_government_contract_spending",
"id"	"territories": [],
],	"default_time": null,
"temporal_granularity":"year"	"description": "Spending by Australian Federal Government agencies on goods and services from 2007 onwards.",
},	"schema_version": "2011-12-07",
"mapping":{	"label": "Australian Federal Government Contract Spending"
"category": {
"fields": [ {
"column": "category",
"datatype": "string",
"name": "label"
} ],

"type": "classifier",
"description": "",
"taxonomy": "unspsc",
"label": "Contract Goods/Services Category"
},	},
"from":{	"mapping": {
"fields":[	"category": {
{	"description": "Contract Goods/Services Category",
"column":"agencyABN",	"datatype": "string",
"datatype":"string",	"label": "category",
"name":"id"	"column": "cat1",
},	"type": "attribute",
{	"dimension": "category"
"column":"agencyName",	},
"datatype":"string",	"sourceurl": {
"name":"label"	"description": "Source Document URL",
}	"datatype": "string",
],	"label": "sourceurl",
"type":"entity",	"column": "sourceurl",
"description":"",	"type": "attribute",
"label":"Government Agency Australian Business Number"	"dimension": "sourceurl"
},	},
"description":{	"from": {
"column":"description",	"attributes": {
"datatype":"string",	"name": {
"type":"value",	"column": "agencyABN",
"description":"Contract as described by agency",	"datatype": "id",
"label":"Contract Description"	"type": "id",
},	"description": null
"to":{	},
"fields":[	"label": {
{	"column": "agencyName",
"column":"supplierName",	"datatype": "string",
"datatype":"string",	"type": "attribute",
"name":"label"	"description": null
},	}
{	},
"column":"supplierID",	"label": "Government Agency",
"datatype":"string",	"type": "compound",
"name":"id"	"dimension": "from",
}	"description": "Government Agency"
],	},
"type":"entity",	"description": {
"description":"Government Supplier",	"description": null,
"label":"Supplier Name"	"datatype": "string",
},	"label": "description",
"currency":{	"column": "description",
"default_value":"AUD",	"type": "attribute",
"description":"All entries in AUD, foreign transactions are converted at the time of their payment",	"dimension": "description"
"column":"",	},
"label":"",	"publishdate": {
"datatype":"currency",	"description": null,
"type":"value"	"format": null,
},	"datatype": "date",
"amount":{	"label": "publishDate",
"column":"value",	"column": "publishDate",
"datatype":"float",	"type": "date",
"type":"value",	"dimension": "publishdate"
"description":"The total value of the contract including all variations/amendments/extensions",	},
"label":"Contract Value"	"to": {
},	"attributes": {
"time":{	"name": {
"column":"contractStart",	"column": "supplierid",
"datatype":"date",	"datatype": "id",
"type":"value",	"type": "id"
"description":"Start of the contract period (goods or services being received)",	},
"label":"Contract Start Date"	"label": {
},	"column": "supplierName",
"id":{	"datatype": "string",
"column": "CNID",

<?php	<?php

error_reporting(E_ALL);	error_reporting(E_ALL);

include_once("../lib/common.inc.php");	include_once("../lib/common.inc.php");

$unspscresult = $conn->prepare('select * from "UNSPSCcategories";');	$unspscresult = $conn->prepare('select * from "UNSPSCcategories";');
$unspscresult->execute();	$unspscresult->execute();
foreach ($unspscresult->fetchAll() as $row) {	foreach ($unspscresult->fetchAll() as $row) {
$unspsc[$row['Title']] = $row['UNSPSC'];	$unspsc[$row['Title']] = $row['UNSPSC'];
// some Australian spellings	// some Australian spellings
$isiz = str_replace("iz", "is", $row['Title']);	$isiz = str_replace("iz", "is", $row['Title']);
$unspsc[$isiz] = $row['UNSPSC'];	$unspsc[$isiz] = $row['UNSPSC'];
$filfill = str_replace("fill", "fil", $row['Title']);	$filfill = str_replace("fill", "fil", $row['Title']);
$unspsc[$filfill] = $row['UNSPSC'];	$unspsc[$filfill] = $row['UNSPSC'];
$defence = str_replace("efense", "efence", $row['Title']);	$defence = str_replace("efense", "efence", $row['Title']);
$unspsc[$defence] = $row['UNSPSC'];	$unspsc[$defence] = $row['UNSPSC'];
$armor = str_replace("rmored", "rmoured", $row['Title']);	$armor = str_replace("rmored", "rmoured", $row['Title']);
$unspsc[$armor] = $row['UNSPSC'];	$unspsc[$armor] = $row['UNSPSC'];
$erre = str_replace("er", "re", $row['Title']);	$erre = str_replace("er", "re", $row['Title']);
$unspsc[$erre] = $row['UNSPSC'];	$unspsc[$erre] = $row['UNSPSC'];
$center = str_replace("center", "centre", $row['Title']);	$center = str_replace("center", "centre", $row['Title']);
$unspsc[$center] = $row['UNSPSC'];	$unspsc[$center] = $row['UNSPSC'];
$accessory = str_replace("accesor", "accessor", $row['Title']);	$accessory = str_replace("accesor", "accessor", $row['Title']);
$unspsc[$accessory] = $row['UNSPSC'];	$unspsc[$accessory] = $row['UNSPSC'];
$lyslyz = str_replace("lyz", "lys", $row['Title']);	$lyslyz = str_replace("lyz", "lys", $row['Title']);
$unspsc[$lyslyz] = $row['UNSPSC'];	$unspsc[$lyslyz] = $row['UNSPSC'];
$tire = str_replace("ire", "yre", $row['Title']);	$tire = str_replace("ire", "yre", $row['Title']);
$unspsc[$tire] = $row['UNSPSC'];	$unspsc[$tire] = $row['UNSPSC'];

$pe = str_replace("pe", "pae", $row['Title']);	$pe = str_replace("pe", "pae", $row['Title']);
$unspsc[$pe] = $row['UNSPSC'];	$unspsc[$pe] = $row['UNSPSC'];
$ane = str_replace("ane", "anae", $row['Title']);	$ane = str_replace("ane", "anae", $row['Title']);
$unspsc[$ane] = $row['UNSPSC'];	$unspsc[$ane] = $row['UNSPSC'];
$airo = str_replace("airplane", "aeroplane", $row['Title']);	$airo = str_replace("airplane", "aeroplane", $row['Title']);
$unspsc[$airo] = $row['UNSPSC'];	$unspsc[$airo] = $row['UNSPSC'];
// some divergence from standard	// some divergence from standard
$forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']);	$forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']);
$unspsc[$forensicit] = $row['UNSPSC'];	$unspsc[$forensicit] = $row['UNSPSC'];
$powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']);	$powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']);
$unspsc[$powercable] = $row['UNSPSC'];	$unspsc[$powercable] = $row['UNSPSC'];
$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);	$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);
$unspsc[$tobacco] = $row['UNSPSC'];	$unspsc[$tobacco] = $row['UNSPSC'];
$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);	$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
$unspsc[$architect] = $row['UNSPSC'];	$unspsc[$architect] = $row['UNSPSC'];
	$powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']);
	$unspsc[$powercable] = $row['UNSPSC'];
	$forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
	$unspsc[$architect] = $row['UNSPSC'];
// some just plain wrong	// some just plain wrong
$noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);	$noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);
$unspsc[$noOilRigs] = $row['UNSPSC'];	$unspsc[$noOilRigs] = $row['UNSPSC'];
}	}
$query = 'SELECT "CNID","category","value"	$query = 'SELECT "CNID","category","value"
FROM contractnotice	FROM contractnotice
WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0';	WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0';
$emptycatresult = $conn->prepare($query);	$emptycatresult = $conn->prepare($query);
$missing = Array();	$missing = Array();
$emptycatresult->execute();	$emptycatresult->execute();
foreach ($emptycatresult->fetchAll() as $record) {	foreach ($emptycatresult->fetchAll() as $record) {

if (!isset($unspsc[$record['category']]) \|\| $unspsc[$record['category']] == "") {	if (!isset($unspsc[$record['category']]) \|\| $unspsc[$record['category']] == "") {
$missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value'];	$missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value'];
// echo "<br>\n Category not found for: \n";	// echo "<br>\n Category not found for: \n";
// print_r($record);	// print_r($record);
} else {	} else {
$result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" =	$result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" =
\'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';');	\'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';');
if ($result) {	if ($result) {
echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n";	echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n";
} else {	} else {
echo "error<br>";	echo "error<br>";
print_r($conn->errorInfo());	print_r($conn->errorInfo());
}	}
}	}
}	}
asort($missing, SORT_NUMERIC);	asort($missing, SORT_NUMERIC);
print_r($missing);	print_r($missing);
?>	?>