Port data exporter to postgres
--- a/admin/partialdata/scrapesingle.php
+++ b/admin/partialdata/scrapesingle.php
@@ -1,4 +1,6 @@
<?php
+include_once ("../../lib/common.inc.php");
+
$cnid = 1234;
// http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/
@@ -800,12 +802,80 @@
</body>
</html> ');
-
+ $datamapping0711 = array(
+ "Agency" => "agencyName",
+ "Parent CN ID" => "parentCN",
+ "CN ID" => "CNID",
+ "Publish Date" => "publishDate",
+ "Amendment Date" => "amendDate",
+ "Status" => "",
+ "StartDate" => "contractStart",
+ "EndDate" => "contractEnd",
+ "Contract Value (AUD)" => "value",
+ "Description" => "description",
+ "Agency Reference ID" => "agencyID",
+ "Category" => "category",
+ "Procurement Method" => "procurementMethod",
+ "ATM ID" => "atmID",
+ "SON ID" => "SONID",
+ "Confidentiality - Contract" => "confidentialityContract",
+ "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
+ "Confidentiality - Outputs" => "confidentialityOutputs",
+ "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
+ "Consultancy" => "consultancy",
+ "Consultancy Reason(s)" => "consultancyReason",
+ "Amendment Reason" => "amendmentReason",
+ "Name" => "supplierName",
+ "Postal Address" => "supplierAddress",
+ "Town/City" => "supplierCity",
+ "Postcode" => "supplierPostcode",
+ "Country" => "supplierCountry",
+ "ABN Exempt" => "supplierABNExempt",
+ "ABN" => "supplierABN",
+ "Branch" => "contactBranch",
+ "Division" => "contactDivision",
+ "Office Postcode" => "contactPostcode"
+ );
+$cnFields = Array();
foreach(pq('tr') as $tr) {
$tra = dom_to_array($tr);
- $tra['th'] = trim(str_replace("/th>","",$tra['th']));
-echo $tra['th']. " = " .trim(print_r($tra['td'],true))."<br>\n";
-
+ $fieldName = trim(str_replace("/th>","",$tra['th']));
+ $fieldValue = trim(print_r($tra['td'],true));
+ if ($fieldName == "State/Territory" || $fieldName == "Contact Name" || $fieldName == "Contact Phone") {
+ // do nothing
+ } else if ($fieldName == "Contract Period") {
+ $contractPeriod = explode("to",$fieldValue);
+ $cnFields["contractStart"] = trim($contractPeriod[0]);
+ $cnFields["contractEnd"] = trim($contractPeriod[1]);
+ } else {
+ $fieldName = $datamapping0711[$fieldName];
+ if ($fieldName == "parentCN" || $fieldName == "CNID") {
+ $fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
+ $fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
+ } elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
+ $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($fieldValue));
+ }
+echo $fieldName. " = " .$fieldValue."<br>\n";
+$cnFields[$fieldName] = $fieldValue;
+ }
}
-
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
+ for($key = 0; $key < sizeof($cnFields); $key++) {
+ $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+ }
+ $contractNoticeInsertQ.= ");";
+ echo $contractNoticeInsertQ;
+ //$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+
+$contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
+$count = 0;
+ foreach ($cnFields as $key => $f) {
+
+ $count++;
+ $contractNoticeUpdateQ.= $key."=? ".($count >= sizeof($cnFields) ? "" : ", ");
+ }
+ $contractNoticeUpdateQ.= " WHERE CNID=?;";
+ $cnFields[] = $cnFields["CNID"];
+ echo $contractNoticeUpdateQ;
+ $contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
?>
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -3,28 +3,34 @@
include_once("./lib/common.inc.php");
setlocale(LC_CTYPE, 'C');
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
-$result = mysql_query('
-SELECT CNID,contractnotice.agencyName,agency.abn as agencyABN,DATE(publishDate),contractStart,contractEnd,value,description,procurementMethod,category,categoryUNSPSC
-,supplierABN,supplierName,supplierCity,supplierPostcode,supplierCountry,contactPostcode,
+$query = $conn->prepare('
+SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",DATE("publishDate"),"contractStart","contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC"
+,"supplierABN","supplierName","supplierCity","supplierPostcode","supplierCountry","contactPostcode",
(
- case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID
+ case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
-concat("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN",CNID) as sourceURL
-FROM `contractnotice` join agency on contractnotice.agencyName=agency.agencyName where childCN = 0');
-if (!$result) die('Couldn\'t fetch records');
-$num_fields = mysql_num_fields($result);
+(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0 limit 5');
+$query->execute();
+if (!$query) {
+ databaseError($conn->errorInfo());
+die('Couldn\'t fetch records');
+}
+
+$num_fields = $query->columnCount();
$headers = array();
for ($i = 0; $i < $num_fields; $i++) {
- $headers[] = mysql_field_name($result , $i);
+ $meta = $query->getColumnMeta($i);
+ $headers[] = $meta['name'];
}
$fp = fopen('php://output', 'w');
-if ($fp && $result) {
+if ($fp && $query) {
header('Content-Type: text/csv');
header('Content-Disposition: attachment; filename="export.'.date("c").'.csv"');
header('Pragma: no-cache');
header('Expires: 0');
fputcsv($fp, $headers);
- while ($row = mysql_fetch_assoc($result)) {
+foreach ($query->fetchAll(PDO::FETCH_NUM) as $row) {
foreach ($row as &$colvalue) {
$colvalue = preg_replace( '/[^[:print:]]/', '',
utf8_encode($colvalue));