Use cursor to export data to use less memory
[contractdashboard.git] / exportData.csv.php
blob:a/exportData.csv.php -> blob:b/exportData.csv.php
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -3,46 +3,62 @@
 include_once("./lib/common.inc.php");
 setlocale(LC_CTYPE, 'C');
 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
+
+$unspsc = Array();
+$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';');
+$unspscresult->execute();
+foreach ($unspscresult->fetchAll() as $row) {
+    $unspsc[$row['UNSPSC']] = $row['Title'];
+}
+/*SELECT count(*), extract("week" from "publishDate") as week,extract("year" from "publishDate") as year
+FROM contractnotice   
+where "childCN" is null group by week, year order by year, week*/
 $query = $conn->prepare('
-SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", 
- "supplierABN","supplierName",
-(
- case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
- 
+SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN",
+EXTRACT(EPOCH FROM "publishDate") as "publishDate",
+EXTRACT(EPOCH FROM "contractStart") as "contractStart",
+EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",
+value,description,category,
+"supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
 (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL 
-FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0'
-, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
-// "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
-// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3")
+FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName"  
+where "childCN" is null'
+        , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
 $query->execute();
-if (!$query) {
-    databaseError($conn->errorInfo());
-die('Couldn\'t fetch records');
+$errors = $conn->errorInfo();
+if ($errors[2] != "") {
+    die("Export terminated, db error" . print_r($errors, true));
 }
 
 $num_fields = $query->columnCount();
-$headers = array();
-for ($i = 0; $i < $num_fields; $i++) {
+$headers = Array();
+for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header
     $meta = $query->getColumnMeta($i);
     $headers[] = $meta['name'];
 }
 $fp = fopen('php://output', 'w');
 if ($fp && $query) {
     header('Content-Type: text/csv');
-    header('Content-Disposition: attachment; filename="export.'.date("c").'.csv"');
+    header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
     header('Pragma: no-cache');
     header('Expires: 0');
     fputcsv($fp, $headers);
- while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
-	foreach ($row as $key => &$colvalue) {
-       
-			$colvalue =  preg_replace( '/[^[:print:]]/', '',
-			 utf8_encode($colvalue));
-             if ($headers[$key] == "publishDate" || $headers[$key] ==  "contractStart" 
-             || $headers[$key] ==  "contractEnd") {
-                 $colvalue = date("Y-m-d",$colvalue);
-             }
-		}
+    while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
+        foreach ($row as $key => &$colvalue) {
+
+            $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($colvalue));
+            if ($headers[$key] == "publishDate" || $headers[$key] == "contractStart"
+                    || $headers[$key] == "contractEnd") {
+                $colvalue = date("Y-m-d", $colvalue);
+            }
+           /* if ($headers[$key] == "CNID") {
+                $colvalue = str_replace("A","", $colvalue);
+}*/
+            if ($headers[$key] == "cat1" || $headers[$key] == "cat2"
+                    || $headers[$key] == "cat3") {
+                $colvalue = $unspsc[$colvalue];
+            }
+        }
         fputcsv($fp, array_values($row));
     }
     die;