Use cursor to export data to use less memory
--- a/admin/partialdata/importamendments.php
+++ b/admin/partialdata/importamendments.php
@@ -1,6 +1,13 @@
<?php
include_once ("../../lib/common.inc.php");
-
+/*
+update contractnotice set "supplierABN" = a."supplierABN"
+from contractnotice as cn inner join (select "supplierABN",
+"supplierName" from contractnotice where "supplierABN"
+IS NOT NULL and "supplierABN" != 0) as a on
+cn."supplierName" = a."supplierName" where
+cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN"
+IS NULL or contractnotice."supplierABN" = 0) */
// http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/
require('phpQuery-onefile.php');
@@ -313,3 +320,4 @@
}
?>
+
--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -10,12 +10,16 @@
// some Australian spellings
$isiz = str_replace("iz","is",$row['Title']);
$unspsc[$isiz] = $row['UNSPSC'];
+ $filfill = str_replace("fill","fil",$row['Title']);
+ $unspsc[$filfill] = $row['UNSPSC'];
$defence = str_replace("efense","efence",$row['Title']);
$unspsc[$defence] = $row['UNSPSC'];
$armor = str_replace("rmored","rmoured",$row['Title']);
$unspsc[$armor] = $row['UNSPSC'];
- $center = str_replace("enter","entre",$row['Title']);
- $unspsc[$center] = $row['UNSPSC'];
+ $erre = str_replace("er","re",$row['Title']);
+ $unspsc[$erre] = $row['UNSPSC'];
+ $lyslyz = str_replace("lyz","lys",$row['Title']);
+ $unspsc[$lyslyz] = $row['UNSPSC'];
// some divergence from standard
$tobacco = str_replace("Food Beverage and Tobacco Products","Food and Beverage Products",$row['Title']);
$unspsc[$tobacco] = $row['UNSPSC'];
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -5,12 +5,14 @@
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
$query = $conn->prepare('
SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC",
- "supplierABN","supplierName","supplierCity","supplierPostcode","supplierCountry","contactPostcode",
+ "supplierABN","supplierName",
(
case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
-FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0');
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0'
+, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
+// "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3")
$query->execute();
if (!$query) {
@@ -31,7 +33,7 @@
header('Pragma: no-cache');
header('Expires: 0');
fputcsv($fp, $headers);
-foreach ($query->fetchAll(PDO::FETCH_NUM) as $row) {
+ while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
foreach ($row as $key => &$colvalue) {
$colvalue = preg_replace( '/[^[:print:]]/', '',