Support partial data amendments with multi line descriptions
--- a/admin/import.php
+++ b/admin/import.php
@@ -261,10 +261,10 @@
}
// run post import data processing
// cn
-//include ("linkAmendments.php");
+include ("linkAmendments.php");
include ("updateUNSPSC.php");
// agency
-include ("setAgencyStatus.php");
-include ("setAgencyURLABN.php");
+//include ("setAgencyStatus.php");
+//include ("setAgencyURLABN.php");
?>
--- a/admin/partialdata/import.php
+++ b/admin/partialdata/import.php
@@ -136,7 +136,7 @@
if ($dhandle) {
// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {
- if (($fname != '.') && ($fname != '..')) {
+ if (($fname != '.') && ($fname != '..') && (!isset($_REQUEST["filter"]) || strpos($fname,$_REQUEST["filter"]) != false)) {
echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice");
}
--- a/admin/partialdata/importamendments.php
+++ b/admin/partialdata/importamendments.php
@@ -1,6 +1,13 @@
<?php
include_once ("../../lib/common.inc.php");
-
+/*
+update contractnotice set "supplierABN" = a."supplierABN"
+from contractnotice as cn inner join (select "supplierABN",
+"supplierName" from contractnotice where "supplierABN"
+IS NOT NULL and "supplierABN" != 0) as a on
+cn."supplierName" = a."supplierName" where
+cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN"
+IS NULL or contractnotice."supplierABN" = 0) */
// http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/
require('phpQuery-onefile.php');
@@ -10,6 +17,8 @@
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, 45);
+curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
+curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
$page = curl_exec($ch);
if (curl_errno($ch)) {
echo "<font color=red> Database temporarily unavailable: ";
@@ -212,6 +221,10 @@
$cnFields[$fieldName] = $fieldValue;
}
}
+if (isset($cnFields[""])) {
+ $cnFields["description"] .= $cnFields[""];
+ unset($cnFields[""]);
+}
$cnFields["importFile"] = $url;
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
for($key = 0; $key < sizeof($cnFields); $key++) {
@@ -297,7 +310,7 @@
if ($dhandle) {
// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {
- if (($fname != '.') && ($fname != '..')) {
+ if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) {
echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice");
}
@@ -313,3 +326,4 @@
}
?>
+
--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -10,12 +10,16 @@
// some Australian spellings
$isiz = str_replace("iz","is",$row['Title']);
$unspsc[$isiz] = $row['UNSPSC'];
+ $filfill = str_replace("fill","fil",$row['Title']);
+ $unspsc[$filfill] = $row['UNSPSC'];
$defence = str_replace("efense","efence",$row['Title']);
$unspsc[$defence] = $row['UNSPSC'];
$armor = str_replace("rmored","rmoured",$row['Title']);
$unspsc[$armor] = $row['UNSPSC'];
- $center = str_replace("enter","entre",$row['Title']);
- $unspsc[$center] = $row['UNSPSC'];
+ $erre = str_replace("er","re",$row['Title']);
+ $unspsc[$erre] = $row['UNSPSC'];
+ $lyslyz = str_replace("lyz","lys",$row['Title']);
+ $unspsc[$lyslyz] = $row['UNSPSC'];
// some divergence from standard
$tobacco = str_replace("Food Beverage and Tobacco Products","Food and Beverage Products",$row['Title']);
$unspsc[$tobacco] = $row['UNSPSC'];
--- /dev/null
+++ b/australian_federal_government_contract_spending.json
@@ -1,1 +1,125 @@
+{
+ "dataset":{
+ "name":"australian_federal_government_contract_spending",
+ "label":"Australian Federal Government Contract Spending",
+ "description":"Spending by Australian Federal Government agencies on goods and services from 2007 onwards.",
+ "currency":"AUD",
+ "unique_keys":[
+ "id"
+ ],
+ "temporal_granularity":"year"
+ },
+ "mapping":{
+ "category": {
+ "fields": [ {
+ "column": "category",
+ "datatype": "string",
+ "name": "label"
+ } ],
+ "type": "classifier",
+ "description": "",
+ "taxonomy": "unspsc",
+ "label": "Contract Goods/Services Category"
+ },
+ "from":{
+ "fields":[
+ {
+ "column":"agencyABN",
+ "datatype":"string",
+ "name":"id"
+ },
+ {
+ "column":"agencyName",
+ "datatype":"string",
+ "name":"label"
+ }
+ ],
+ "type":"entity",
+ "description":"",
+ "label":"Government Agency Australian Business Number"
+ },
+ "description":{
+ "column":"description",
+ "datatype":"string",
+ "type":"value",
+ "description":"Contract as described by agency",
+ "label":"Contract Description"
+ },
+ "to":{
+ "fields":[
+ {
+ "column":"supplierName",
+ "datatype":"string",
+ "name":"label"
+ },
+ {
+ "column":"supplierID",
+ "datatype":"string",
+ "name":"id"
+ }
+ ],
+ "type":"entity",
+ "description":"Government Supplier",
+ "label":"Supplier Name"
+ },
+ "currency":{
+ "default_value":"AUD",
+ "description":"All entries in AUD, foreign transactions are converted at the time of their payment",
+ "column":"",
+ "label":"",
+ "datatype":"currency",
+ "type":"value"
+ },
+ "amount":{
+ "column":"value",
+ "datatype":"float",
+ "type":"value",
+ "description":"The total value of the contract including all variations/amendments/extensions",
+ "label":"Contract Value"
+ },
+ "time":{
+ "column":"contractStart",
+ "datatype":"date",
+ "type":"value",
+ "description":"Start of the contract period (goods or services being received)",
+ "label":"Contract Start Date"
+ },
+ "id":{
+"column": "CNID",
+ "datatype": "string",
+ "name": "label",
+ "type": "value",
+ "description": "",
+ "label": "Contract Notice ID"
+ }
+ },
+
+"views": [
+ {
+ "name": "default",
+ "entity": "dataset",
+ "label": "Default breakdown by Category",
+ "dimension": "dataset",
+ "breakdown": "category",
+ "filters" : {"name": "australian_federal_government_contract_spending" }
+ },
+ {
+ "name":"default",
+ "entity": "classifier",
+ "label":"Breakdown of Spending Areas by Supplier Company",
+ "dimension": "category",
+ "breakdown": "to",
+ "filters": {"taxonomy": "unspsc"}
+ },
+ {
+ "name":"default",
+ "entity": "classifier",
+ "label":"Breakdown of Spending Areas by Government Agency",
+ "dimension": "category",
+ "breakdown": "from",
+ "filters": {"taxonomy": "unspsc"}
+ }
+]
+}
+
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -3,31 +3,45 @@
include_once("./lib/common.inc.php");
setlocale(LC_CTYPE, 'C');
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
-$result = mysql_query('
-SELECT CNID,contractnotice.agencyName,agency.abn as agencyABN,DATE(publishDate),contractStart,contractEnd,value,description,procurementMethod,category,categoryUNSPSC
-,supplierABN,supplierName,supplierCity,supplierPostcode,supplierCountry,contactPostcode,
+$query = $conn->prepare('
+SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC",
+ "supplierABN","supplierName",
(
- case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID
+ case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
-concat("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN",CNID) as sourceURL
-FROM `contractnotice` join agency on contractnotice.agencyName=agency.agencyName where childCN = 0');
-if (!$result) die('Couldn\'t fetch records');
-$num_fields = mysql_num_fields($result);
+(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0'
+, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
+// "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
+// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3")
+$query->execute();
+if (!$query) {
+ databaseError($conn->errorInfo());
+die('Couldn\'t fetch records');
+}
+
+$num_fields = $query->columnCount();
$headers = array();
for ($i = 0; $i < $num_fields; $i++) {
- $headers[] = mysql_field_name($result , $i);
+ $meta = $query->getColumnMeta($i);
+ $headers[] = $meta['name'];
}
$fp = fopen('php://output', 'w');
-if ($fp && $result) {
+if ($fp && $query) {
header('Content-Type: text/csv');
header('Content-Disposition: attachment; filename="export.'.date("c").'.csv"');
header('Pragma: no-cache');
header('Expires: 0');
fputcsv($fp, $headers);
- while ($row = mysql_fetch_assoc($result)) {
- foreach ($row as &$colvalue) {
+ while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
+ foreach ($row as $key => &$colvalue) {
+
$colvalue = preg_replace( '/[^[:print:]]/', '',
utf8_encode($colvalue));
+ if ($headers[$key] == "publishDate" || $headers[$key] == "contractStart"
+ || $headers[$key] == "contractEnd") {
+ $colvalue = date("Y-m-d",$colvalue);
+ }
}
fputcsv($fp, array_values($row));
}
--- a/lib/graphs.inc.php
+++ b/lib/graphs.inc.php
@@ -7,6 +7,7 @@
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script>
<script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script>
+ <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.stack.js"></script>
';
$includedFlot = true;
@@ -23,8 +24,9 @@
$(function () {
var d1 = [];
+ var d2 = [];
<?php
-$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999) as a group by cnid order by cnid';
+$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is null) as a group by cnid order by cnid';
$query = $conn->prepare($query);
$query->execute();
if (!$query) {
@@ -35,6 +37,18 @@
foreach ($query->fetchAll() as $delta) {
echo "d1.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
+};
+$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid';
+$query = $conn->prepare($query);
+$query->execute();
+if (!$query) {
+ databaseError($conn->errorInfo());
+ return Array();
+}
+
+foreach ($query->fetchAll() as $delta) {
+
+ echo "d2.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
};
?>
@@ -47,10 +61,20 @@
},
bars: { show: true }
},
+ {
+ data: d2,
+series: {
+ lines: { show: true },
+ points: { show: true }
+ },
+ bars: { show: true }
+ },
];
var options =
{
-
+series: {
+ stack: true,
+ },
grid: { hoverable: true, clickable: true, labelMargin: 17 },
selection: { mode: "x" }
};