projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Include textual subcategories in openspending/csv export
Include textual subcategories in openspending/csv export

file:a/admin/updateUNSPSC.php -> file:b/admin/updateUNSPSC.php
<?php <?php
error_reporting(E_ALL); error_reporting(E_ALL);
include_once("../lib/common.inc.php"); include_once("../lib/common.inc.php");
$unspscresult = $conn->prepare('select * from "UNSPSCcategories";'); $unspscresult = $conn->prepare('select * from "UNSPSCcategories";');
$unspscresult->execute(); $unspscresult->execute();
foreach ($unspscresult->fetchAll() as $row) { foreach ($unspscresult->fetchAll() as $row) {
$unspsc[$row['Title']] = $row['UNSPSC']; $unspsc[$row['Title']] = $row['UNSPSC'];
// some Australian spellings // some Australian spellings
$isiz = str_replace("iz", "is", $row['Title']); $isiz = str_replace("iz", "is", $row['Title']);
$unspsc[$isiz] = $row['UNSPSC']; $unspsc[$isiz] = $row['UNSPSC'];
$filfill = str_replace("fill", "fil", $row['Title']); $filfill = str_replace("fill", "fil", $row['Title']);
$unspsc[$filfill] = $row['UNSPSC']; $unspsc[$filfill] = $row['UNSPSC'];
$defence = str_replace("efense", "efence", $row['Title']); $defence = str_replace("efense", "efence", $row['Title']);
$unspsc[$defence] = $row['UNSPSC']; $unspsc[$defence] = $row['UNSPSC'];
$armor = str_replace("rmored", "rmoured", $row['Title']); $armor = str_replace("rmored", "rmoured", $row['Title']);
$unspsc[$armor] = $row['UNSPSC']; $unspsc[$armor] = $row['UNSPSC'];
$erre = str_replace("er", "re", $row['Title']); $erre = str_replace("er", "re", $row['Title']);
$unspsc[$erre] = $row['UNSPSC']; $unspsc[$erre] = $row['UNSPSC'];
  $tire = str_replace("ire", "yre", $row['Title']);
  $unspsc[$tire] = $row['UNSPSC'];
  $aeroplane = str_replace("airplane","aeroplane", $row['Title']);
  $unspsc[$aeroplane] = $row['UNSPSC'];
$lyslyz = str_replace("lyz", "lys", $row['Title']); $lyslyz = str_replace("lyz", "lys", $row['Title']);
$unspsc[$lyslyz] = $row['UNSPSC']; $unspsc[$lyslyz] = $row['UNSPSC'];
// some divergence from standard // some divergence from standard
$tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);
$unspsc[$tobacco] = $row['UNSPSC']; $unspsc[$tobacco] = $row['UNSPSC'];
$architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
  $unspsc[$architect] = $row['UNSPSC'];
  $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']);
  $unspsc[$powercable] = $row['UNSPSC'];
  $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);
$unspsc[$architect] = $row['UNSPSC']; $unspsc[$architect] = $row['UNSPSC'];
// some just plain wrong // some just plain wrong
$noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);
$unspsc[$noOilRigs] = $row['UNSPSC']; $unspsc[$noOilRigs] = $row['UNSPSC'];
} }
$query = 'SELECT "CNID","category","value" $query = 'SELECT "CNID","category","value"
FROM contractnotice FROM contractnotice
WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0'; WHERE "categoryUNSPSC" IS NULL OR "categoryUNSPSC" = 0';
$emptycatresult = $conn->prepare($query); $emptycatresult = $conn->prepare($query);
$missing = Array(); $missing = Array();
$emptycatresult->execute(); $emptycatresult->execute();
foreach ($emptycatresult->fetchAll() as $record) { foreach ($emptycatresult->fetchAll() as $record) {
if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") { if (!isset($unspsc[$record['category']]) || $unspsc[$record['category']] == "") {
$missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value']; $missing[$record['category']] = (isset($missing[$record['category']]) ? $missing[$record['category']] : 0) + $record['value'];
// echo "<br>\n Category not found for: \n"; // echo "<br>\n Category not found for: \n";
// print_r($record); // print_r($record);
} else { } else {
$result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" = $result = $conn->exec('UPDATE contractnotice SET "categoryUNSPSC" =
\'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';'); \'' . $unspsc[$record['category']] . '\' where "CNID" = \'' . $record['CNID'] . '\';');
if ($result) { if ($result) {
echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n"; echo $record['CNID'] . " set to " . ($unspsc[$record['category']]) . " <br>\n";
} else { } else {
echo "error<br>"; echo "error<br>";
print_r($conn->errorInfo()); print_r($conn->errorInfo());
} }
} }
} }
asort($missing, SORT_NUMERIC); asort($missing, SORT_NUMERIC);
print_r($missing); print_r($missing);
?> ?>
   
file:a/exportData.csv.php -> file:b/exportData.csv.php
<?php <?php
   
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
setlocale(LC_CTYPE, 'C'); setlocale(LC_CTYPE, 'C');
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
   
  $unspsc = Array();
  $unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';');
  $unspscresult->execute();
  foreach ($unspscresult->fetchAll() as $row) {
  $unspsc[$row['UNSPSC']] = $row['Title'];
  }
   
$query = $conn->prepare(' $query = $conn->prepare('
SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",
  EXTRACT(EPOCH FROM "publishDate") as "publishDate",
  EXTRACT(EPOCH FROM "contractStart") as "contractStart",
  EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",
  value,description,"procurementMethod",category,"categoryUNSPSC",
  (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "cat1",
  (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "cat2",
  (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text) as "cat3",
"supplierABN","supplierName", "supplierABN","supplierName",
( (
case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" is null' FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName"
  where "childCN" is null'
, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
// "supplierCity","supplierPostcode","supplierCountry","contactPostcode", // "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3") //
$query->execute(); $query->execute();
$errors = $conn->errorInfo(); $errors = $conn->errorInfo();
if ($errors[2] != "") { if ($errors[2] != "") {
die("Export terminated, db error" . print_r($errors, true)); die("Export terminated, db error" . print_r($errors, true));
} }
   
$num_fields = $query->columnCount(); $num_fields = $query->columnCount();
$headers = array(); $headers = Array();
for ($i = 0; $i < $num_fields; $i++) { for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header
$meta = $query->getColumnMeta($i); $meta = $query->getColumnMeta($i);
$headers[] = $meta['name']; $headers[] = $meta['name'];
} }
$fp = fopen('php://output', 'w'); $fp = fopen('php://output', 'w');
if ($fp && $query) { if ($fp && $query) {
header('Content-Type: text/csv'); header('Content-Type: text/csv');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache'); header('Pragma: no-cache');
header('Expires: 0'); header('Expires: 0');
fputcsv($fp, $headers); fputcsv($fp, $headers);
while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) { while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
foreach ($row as $key => &$colvalue) { foreach ($row as $key => &$colvalue) {
   
$colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($colvalue)); $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($colvalue));
if ($headers[$key] == "publishDate" || $headers[$key] == "contractStart" if ($headers[$key] == "publishDate" || $headers[$key] == "contractStart"
|| $headers[$key] == "contractEnd") { || $headers[$key] == "contractEnd") {
$colvalue = date("Y-m-d", $colvalue); $colvalue = date("Y-m-d", $colvalue);
} }
  if ($headers[$key] == "cat1" || $headers[$key] == "cat2"
  || $headers[$key] == "cat3") {
  $colvalue = $unspsc[$colvalue];
  }
} }
fputcsv($fp, array_values($row)); fputcsv($fp, array_values($row));
} }
die; die;
} }
?> ?>
   
Unnamed repository; edit this file to name it for gitweb.
GitPHP by Chris Han