From: maxious <maxious@lambdacomplex.org>
Date: Mon, 08 Aug 2011 05:23:40 +0000
Subject: Port data exporter to postgres
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=0bdd84df9780e4b933b01e1dd0d7a6ec1f469155
---
Port data exporter to postgres
---


--- a/admin/partialdata/scrapesingle.php
+++ b/admin/partialdata/scrapesingle.php
@@ -1,4 +1,6 @@
 <?php
+include_once ("../../lib/common.inc.php");
+
 $cnid = 1234;
 // http://www.lastcraft.com/browser_documentation.php
 // http://code.google.com/p/phpquery/
@@ -800,12 +802,80 @@
 </body>
 
 </html> ');
-
+	$datamapping0711 = array(
+		"Agency" => "agencyName",
+		"Parent CN ID" => "parentCN",
+		"CN ID" => "CNID",
+		"Publish Date" => "publishDate",
+		"Amendment Date" => "amendDate",
+		"Status" => "",
+		"StartDate" => "contractStart",
+		"EndDate" => "contractEnd",
+		"Contract Value (AUD)" => "value",
+		"Description" => "description",
+		"Agency Reference ID" => "agencyID",
+		"Category" => "category",
+		"Procurement Method" => "procurementMethod",
+		"ATM ID" => "atmID",
+		"SON ID" => "SONID",
+		"Confidentiality - Contract" => "confidentialityContract",
+		"Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
+		"Confidentiality - Outputs" => "confidentialityOutputs",
+		"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
+		"Consultancy" => "consultancy",
+		"Consultancy Reason(s)" => "consultancyReason",
+		"Amendment Reason" => "amendmentReason",
+		"Name" => "supplierName",
+		"Postal Address" => "supplierAddress",
+		"Town/City" => "supplierCity",
+		"Postcode" => "supplierPostcode",
+		"Country" => "supplierCountry",
+		"ABN Exempt" => "supplierABNExempt",
+		"ABN" => "supplierABN",
+		"Branch" => "contactBranch",
+		"Division" => "contactDivision",
+		"Office Postcode" => "contactPostcode"
+	);
+$cnFields = Array();
 foreach(pq('tr') as $tr) {
 	$tra = dom_to_array($tr);
-	$tra['th'] = trim(str_replace("/th>","",$tra['th']));
-echo $tra['th']. " = " .trim(print_r($tra['td'],true))."<br>\n";
-
+	$fieldName = trim(str_replace("/th>","",$tra['th']));
+	$fieldValue = trim(print_r($tra['td'],true));
+	if ($fieldName == "State/Territory" || $fieldName == "Contact Name" || $fieldName == "Contact Phone") {
+	    // do nothing
+	} else if ($fieldName == "Contract Period") {
+	    $contractPeriod = explode("to",$fieldValue);
+	    $cnFields["contractStart"] =  trim($contractPeriod[0]);
+		$cnFields["contractEnd"] =  trim($contractPeriod[1]);
+	} else {
+	    $fieldName = $datamapping0711[$fieldName];
+	if ($fieldName == "parentCN" || $fieldName == "CNID") {
+							$fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
+							$fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
+	} elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
+							$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($fieldValue));
+						}
+echo $fieldName. " = " .$fieldValue."<br>\n";
+$cnFields[$fieldName] = $fieldValue;
+	}
 }
-
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
+		for($key = 0; $key < sizeof($cnFields); $key++) {
+			$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+		}
+		$contractNoticeInsertQ.= ");";
+		echo $contractNoticeInsertQ;
+		//$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+		
+$contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
+$count = 0;
+		foreach ($cnFields as $key => $f) {
+		    
+			$count++;
+			$contractNoticeUpdateQ.= $key."=? ".($count >= sizeof($cnFields) ? "" : ", ");
+		}
+		$contractNoticeUpdateQ.= " WHERE CNID=?;";
+		$cnFields[] = $cnFields["CNID"];
+		echo $contractNoticeUpdateQ;
+		$contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
 ?>

--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -3,28 +3,34 @@
 include_once("./lib/common.inc.php");
 setlocale(LC_CTYPE, 'C');
 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
-$result = mysql_query('
-SELECT CNID,contractnotice.agencyName,agency.abn as agencyABN,DATE(publishDate),contractStart,contractEnd,value,description,procurementMethod,category,categoryUNSPSC
-,supplierABN,supplierName,supplierCity,supplierPostcode,supplierCountry,contactPostcode,
+$query = $conn->prepare('
+SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",DATE("publishDate"),"contractStart","contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC"
+,"supplierABN","supplierName","supplierCity","supplierPostcode","supplierCountry","contactPostcode",
 (
- case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID
+ case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
  
-concat("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN",CNID) as sourceURL 
-FROM `contractnotice` join agency on contractnotice.agencyName=agency.agencyName where childCN = 0');
-if (!$result) die('Couldn\'t fetch records');
-$num_fields = mysql_num_fields($result);
+(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL 
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0 limit 5');
+$query->execute();
+if (!$query) {
+    databaseError($conn->errorInfo());
+die('Couldn\'t fetch records');
+}
+
+$num_fields = $query->columnCount();
 $headers = array();
 for ($i = 0; $i < $num_fields; $i++) {
-    $headers[] = mysql_field_name($result , $i);
+    $meta = $query->getColumnMeta($i);
+    $headers[] = $meta['name'];
 }
 $fp = fopen('php://output', 'w');
-if ($fp && $result) {
+if ($fp && $query) {
     header('Content-Type: text/csv');
     header('Content-Disposition: attachment; filename="export.'.date("c").'.csv"');
     header('Pragma: no-cache');
     header('Expires: 0');
     fputcsv($fp, $headers);
-    while ($row = mysql_fetch_assoc($result)) {
+foreach ($query->fetchAll(PDO::FETCH_NUM) as $row) {
 	foreach ($row as &$colvalue) {
 			$colvalue =  preg_replace( '/[^[:print:]]/', '',
 			 utf8_encode($colvalue));