Amendment partial record importer
--- /dev/null
+++ b/admin/neo4jimporter/.classpath
@@ -1,1 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" output="target/classes" path="src/main/java"/>
+ <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
+ <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
+ <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
+ <classpathentry kind="output" path="target/classes"/>
+</classpath>
--- /dev/null
+++ b/admin/neo4jimporter/.project
@@ -1,1 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>neo4jimporter</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.eclipse.m2e.core.maven2Nature</nature>
+ </natures>
+</projectDescription>
--- /dev/null
+++ b/admin/neo4jimporter/.settings/org.eclipse.jdt.core.prefs
@@ -1,1 +1,7 @@
+#Sun Aug 07 18:15:32 EST 2011
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
+org.eclipse.jdt.core.compiler.compliance=1.5
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.5
--- /dev/null
+++ b/admin/neo4jimporter/.settings/org.eclipse.m2e.core.prefs
@@ -1,1 +1,6 @@
+#Sun Aug 07 18:14:30 EST 2011
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
--- /dev/null
+++ b/admin/neo4jimporter/pom.xml
@@ -1,1 +1,18 @@
-
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.lambdacomplex.contractdashboard</groupId>
+ <artifactId>neo4jimporter</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.neo4j</groupId>
+ <artifactId>neo4j-kernel</artifactId>
+ <version>1.4</version>
+ </dependency>
+ <dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>9.0-801.jdbc4</version>
+</dependency>
+ </dependencies>
+</project>
--- /dev/null
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -1,1 +1,155 @@
+import java.io.ObjectInputStream.GetField;
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+import org.neo4j.graphdb.DynamicRelationshipType;
+import org.neo4j.kernel.impl.batchinsert.BatchInserter;
+import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
+
+public class Importer {
+
+ public static void main(String[] argv) {
+
+ Map<String,String> props = new HashMap<String, String>();
+ props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes>
+ props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes>
+ // create the batch inserter
+ BatchInserter inserter = new
+ BatchInserterImpl("neo4j-db/",props
+ );
+
+
+
+
+
+ System.out.println("-------- PostgreSQL "
+ + "JDBC Connection Testing ------------");
+
+ try {
+
+ Class.forName("org.postgresql.Driver");
+
+ } catch (ClassNotFoundException e) {
+
+ System.out.println("Where is your PostgreSQL JDBC Driver? "
+ + "Include in your library path!");
+ e.printStackTrace();
+
+ }
+
+ System.out.println("PostgreSQL JDBC Driver Registered!");
+
+ Connection conn = null;
+
+ try {
+
+ conn = DriverManager.getConnection(
+ "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
+ "postgres", "snmc");
+
+ } catch (SQLException e) {
+
+ System.out.println("Connection Failed! Check output console");
+ e.printStackTrace();
+
+ }
+
+ if (conn != null) {
+ System.out.println("You made it, take control your database now!");
+ } else {
+ System.out.println("Failed to make connection!");
+ }
+ try {
+ // Print all warnings
+ for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn
+ .getNextWarning()) {
+ System.out.println("SQL Warning:");
+ System.out.println("State : " + warn.getSQLState());
+ System.out.println("Message: " + warn.getMessage());
+ System.out.println("Error : " + warn.getErrorCode());
+ }
+
+ // Get a statement from the connection
+ Statement stmt = conn.createStatement();
+
+ // Execute the query
+ ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;");
+String previousAgency = "";
+
+ // Loop through the result set
+ while (rs.next()) {
+ long supplier,agency;
+ agency = doHash(rs.getString("agencyName"));
+ if (rs.getString("agencyName") != previousAgency) {
+ if (!inserter.nodeExists(agency)) {
+ Map<String, Object> properties = new HashMap<String, Object>();
+ properties.put("Label", rs.getString("agencyName"));
+ inserter.createNode(agency, properties);
+ }
+ }
+ if (rs.getString("supplierABN") != "0") {
+ supplier = doHash(rs.getString("supplierABN"));
+ } else {
+ supplier = doHash(rs.getString("supplierName"));
+ }
+ // inject some data
+ if (!inserter.nodeExists(supplier)) {
+ Map<String, Object> properties = new HashMap<String, Object>();
+
+ properties.put("Label", rs.getString("supplierName"));
+ inserter.createNode(supplier, properties);
+ }
+
+ inserter.createRelationship(agency, supplier,
+ DynamicRelationshipType.withName("KNOWS"), null);
+
+ }
+ // Close the result set, statement and the connection
+ rs.close();
+ stmt.close();
+ conn.close();
+ } catch (SQLException se) {
+ System.out.println("SQL Exception:");
+
+ // Loop through the SQL Exceptions
+ while (se != null) {
+ System.out.println("State : " + se.getSQLState());
+ System.out.println("Message: " + se.getMessage());
+ System.out.println("Error : " + se.getErrorCode());
+
+ se = se.getNextException();
+ }
+ } catch (Exception e) {
+ System.out.println(e);
+ }
+ // shutdown, makes sure all changes are written to disk
+ inserter.shutdown();
+ }
+
+ static long doHash(String input) {
+ MessageDigest m;
+ try {
+ m = MessageDigest.getInstance("MD5");
+ m.reset();
+ m.update(input.getBytes());
+ byte[] digest = m.digest();
+ return new BigInteger(1, digest).longValue();
+ } catch (NoSuchAlgorithmException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return 0;
+
+ }
+
+
+}
--- /dev/null
+++ b/admin/neo4jimporter/src/main/java/JDBCExample.java
@@ -1,1 +1,50 @@
-
+import java.sql.DriverManager;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+public class JDBCExample {
+
+ public static void main(String[] argv) {
+
+ System.out.println("-------- PostgreSQL "
+ + "JDBC Connection Testing ------------");
+
+ try {
+
+ Class.forName("org.postgresql.Driver");
+
+ } catch (ClassNotFoundException e) {
+
+ System.out.println("Where is your PostgreSQL JDBC Driver? "
+ + "Include in your library path!");
+ e.printStackTrace();
+ return;
+
+ }
+
+ System.out.println("PostgreSQL JDBC Driver Registered!");
+
+ Connection connection = null;
+
+ try {
+
+ connection = DriverManager.getConnection(
+ "jdbc:postgresql://127.0.0.1:5432/contractDashboard", "postgres",
+ "snmc");
+
+ } catch (SQLException e) {
+
+ System.out.println("Connection Failed! Check output console");
+ e.printStackTrace();
+ return;
+
+ }
+
+ if (connection != null) {
+ System.out.println("You made it, take control your database now!");
+ } else {
+ System.out.println("Failed to make connection!");
+ }
+ }
+
+}
Binary files /dev/null and b/admin/neo4jimporter/target/classes/Importer.class differ
Binary files /dev/null and b/admin/neo4jimporter/target/classes/JDBCExample.class differ
--- a/admin/partialdata/01Jan1970to01Jan1970val0to16000.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
+
--- a/admin/partialdata/01Jan1970to01Jan1970valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
+
--- a/admin/partialdata/07Jun2008to09Jun2008valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
+
--- a/admin/partialdata/12May2008to14May2008val0to8000.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
+
--- a/admin/partialdata/14Aug2007to18Aug2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/14Aug2007to22Aug2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/14Aug2007to29Aug2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/14Nov2007to18Nov2007valto.xls
+++ b/admin/partialdata/14Nov2007to18Nov2007valto.xls
@@ -254,7 +254,7 @@
="CN46513" "Secure LAN facilities for ICON RFT project" ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Insurance and retirement services" 06-Nov-07 31-May-08 26290.12 ="" ="MBITS - Managed Business IT Solutions" 15-Nov-07 02:02 PM
="CN46515" "Business Analyst" ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management advisory services" 01-Nov-07 01-Feb-08 14960.00 ="" ="Dialog Information Technology" 15-Nov-07 02:02 PM
="CN46517-A1" "Provision of recruitment services for the 2009 Traineeship Programme." ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Human resources services" 31-Oct-07 29-Feb-08 18053.00 ="" ="Ross Human Directions Limited" 15-Nov-07 02:03 PM
-="CN46518" "To facilitate negotiations and implement Resource Sharing agreements in Australia's Tuna and Billfish fisheries. The consultant activities are as follows:1. Review existing options (research & interpretation)2. Consultation with stakeholders to identify views and positions (facilitation)3. Collate data, analyse data and prepare draft revised agreements between parties (research)4. Consultations to present, discuss and refine agreements between parties (facilitation)5. Further develop/refine agreements and implementation capabilities6. Consultation to finalise agreements and implementation capabilities7. Written report to Government on matters arising from the agreement and implementation.The key output of these activities will be revised recommendations for resource sharing arrangement in the WTBF and the ETBF based on successful stakeholder consultation." ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management advisory services" 25-Oct-07 25-Oct-08 10843.69 ="" ="Ewan Colquhoun, RidgePartners" 15-Nov-07 02:03 PM
+="CN46518" "To facilitate negotiations and implement Resource Sharing agreements in Australia's Tuna and Billfish fisheries. " ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management advisory services" 25-Oct-07 25-Oct-08 10843.69 ="" ="Ewan Colquhoun, RidgePartners" 15-Nov-07 02:03 PM
="CN46519" "Sample Analysis - Foreign Certificate Audit" ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management and Business Professionals and Administrative Services" 30-Oct-07 30-Apr-08 50000.00 ="" ="National Measurement Institute" 15-Nov-07 02:03 PM
="CN46520" "Sample Analysis - Foreign Certificate Audit" ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management and Business Professionals and Administrative Services" 30-Oct-07 30-Apr-08 20000.00 ="" ="Silliker Microtech Pty Ltd" 15-Nov-07 02:03 PM
="CN46521" "Assessing the feasibility of stereo video and evaluating monitoring options for the southern bluefin tuna farm sector" ="Department of Agriculture Fisheries and Forestry" 15-Nov-07 ="Management advisory services" 01-Jul-07 31-Dec-07 22000.00 ="" ="Australian Southern Bluefin Tuna Industry Association" 15-Nov-07 02:03 PM
--- a/admin/partialdata/15Sep2007to16Sep2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/22Aug2007to30Aug2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/29Aug2007to02Sep2007valto.xls
+++ /dev/null
@@ -1,4 +1,1 @@
-Advanced Search
-CN ID Title Agency Publish Date Category Contract Start Date Contract End Date Value (AUD) ATM ID Supplier Name LastUpdated
-There are no results that match your selection.
--- a/admin/partialdata/import.php
+++ b/admin/partialdata/import.php
@@ -1,9 +1,13 @@
<?php
-include_once ("../lib/common.inc.php");
+include_once ("../../lib/common.inc.php");
function processFile($fpath, $tablename)
{
global $conn;
+ echo " ============== $fpath ============== <br>";
+ flush();
$row = 1;
+ $success = 0;
+ $dupes = 0;
$handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix
$datamapping0507 = array(
@@ -16,7 +20,8 @@
"Title" => "description",
"Category" => "category",
"ATM ID" => "atmID",
- "LastUpdated" => "",
+ "Supplier Name" => "supplierName",
+ "LastUpdated" => "amendDate",
"" => ""
);
$headers;
@@ -32,7 +37,7 @@
"value",
"atmID",
"supplierName",
- "LastUpdated"
+ "amendDate"
);
if ($tablename == "contractnotice") {
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
@@ -50,7 +55,7 @@
}
elseif ($row > 3) {
if ($num > count($datamapping0507)) {
- die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
+ die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . "data:" .$num. print_r($data ,true). "mapping:" . count($datamapping0507). print_r($datamapping0507 ,true));
}
$contractNoticeInsert = Array();
$contractNoticeInsert[] = $fpath;
@@ -98,6 +103,7 @@
$contractNoticeInsertQ->execute($contractNoticeInsert);
$errors = $conn->errorInfo();
if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+ $dupes++;
}
elseif ($errors[1] == 0) {
$success++;
@@ -116,6 +122,9 @@
$row++;
}
fclose($handle);
+ echo " $dupes duplicate records<br>";
+ echo " $success records successfully created<br>";
+ flush();
return $success;
}
$path = './';
@@ -129,6 +138,7 @@
while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..')) {
echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
+ processFile($path . $fname, "contractnotice");
}
}
}
@@ -136,11 +146,9 @@
else {
$success = 0;
$fname = $_REQUEST["fname"];
- echo " ============== $fname ============== <br>";
- flush();
+
$success+= processFile($path . $fname, "contractnotice");
- echo "<br> $success records successfully created";
- flush();
+
}
?>
--- /dev/null
+++ b/admin/partialdata/importamendments.php
@@ -1,1 +1,315 @@
-
+<?php
+include_once ("../../lib/common.inc.php");
+
+// http://www.lastcraft.com/browser_documentation.php
+// http://code.google.com/p/phpquery/
+require('phpQuery-onefile.php');
+function getURL($url) {
+ //return file_get_contents($url);
+ $ch = curl_init($url);
+ curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+ curl_setopt($ch, CURLOPT_HEADER, 0);
+ curl_setopt($ch, CURLOPT_TIMEOUT, 45);
+ $page = curl_exec($ch);
+ if (curl_errno($ch)) {
+ echo "<font color=red> Database temporarily unavailable: ";
+ echo curl_errno($ch) . " " . curl_error($ch);
+
+ echo $url;
+ echo "</font><br>";
+ }
+ curl_close($ch);
+ return $page;
+}
+
+
+function getTextFromTHNode($Node, $Text = "") {
+ if ($Node->tagName == null)
+ return $Text.$Node->textContent;
+
+if ($Node->tagName != "td") {
+ $Node = $Node->firstChild;
+ if ($Node != null)
+ $Text = getTextFromTHNode($Node, $Text);
+
+ while($Node->nextSibling != null) {
+ $Text = getTextFromTHNode($Node->nextSibling, $Text);
+ $Node = $Node->nextSibling;
+ }
+}
+ return $Text;
+}
+
+function getTextFromNode($Node, $Text = "") {
+ if ($Node->tagName == null)
+ return $Text.$Node->textContent;
+if ($Node->tagName != "th" && $Node->tagName != "span") {
+ $Node = $Node->firstChild;
+ if ($Node != null)
+ $Text = getTextFromNode($Node, $Text);
+
+ while($Node->nextSibling != null) {
+ $Text = getTextFromNode($Node->nextSibling, $Text);
+ $Node = $Node->nextSibling;
+ }
+}
+ return $Text;
+}
+function dom_to_array($root)
+{
+ $result = array();
+
+ if ($root->hasAttributes())
+ {
+ $attrs = $root->attributes;
+
+ foreach ($attrs as $i => $attr)
+ $result[$attr->name] = $attr->value;
+ }
+
+ $children = $root->childNodes;
+if ($root->childNodes) {
+ if ($children->length == 1)
+ {
+ $child = $children->item(0);
+
+ if ($child->nodeType == XML_TEXT_NODE)
+ {
+ $result['_value'] = $child->nodeValue;
+
+ if (count($result) == 1)
+ return $result['_value'];
+ else
+ return $result;
+ }
+ }
+
+ $group = array();
+
+ for($i = 0; $i < $children->length; $i++)
+ {
+ $child = $children->item($i);
+
+ if (!isset($result[$child->nodeName]))
+ $result[$child->nodeName] = dom_to_array($child);
+ else
+ {
+ if (!isset($group[$child->nodeName]))
+ {
+ $tmp = $result[$child->nodeName];
+ $result[$child->nodeName] = array($tmp);
+ $group[$child->nodeName] = 1;
+ }
+
+ $result[$child->nodeName][] = dom_to_array($child);
+ }
+ }
+}
+
+ return $result;
+}
+
+function importCN($cnid) {
+ global $conn;
+ $CN = str_replace("-A", "00",$cnid);
+ // check if already complete
+ $query = 'Select "parentCN" from contractnotice
+ where "CNID" = :CNID';
+ $query = $conn->prepare($query);
+ $query->bindParam(":CNID", $CN);
+ $query->execute();
+ $r = $query->fetch(PDO::FETCH_ASSOC);
+ if ($r['parentCN'] == NULL) {
+$site = "https://www.tenders.gov.au/";
+$searchResult = phpQuery::newDocument(getURL("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
+//echo "https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid;
+$url = "";
+foreach(pq('a') as $a) {
+ if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) {
+ //echo $a->getAttribute("href");
+ $url = $a->getAttribute("href");
+ break;
+ }
+}
+$cn = phpQuery::newDocument(getURL($site.$url));
+ $datamapping0711 = array(
+ "Agency" => "agencyName",
+ "Parent CN" => "parentCN",
+ "CN ID" => "CNID",
+ "Publish Date" => "publishDate",
+ "Amendment Date" => "amendDate",
+ "Status" => "",
+ "StartDate" => "contractStart",
+ "EndDate" => "contractEnd",
+ "Contract Value (AUD)" => "value",
+ "Description" => "description",
+ "Agency Reference ID" => "agencyID",
+ "Category" => "category",
+ "Procurement Method" => "procurementMethod",
+ "ATM ID" => "atmID",
+ "SON ID" => "SONID",
+ "Confidentiality - Contract" => "confidentialityContract",
+ "Confidentiality Reason(s) - Contract" => "confidentialityContractReason",
+ "Confidentiality - Outputs" => "confidentialityOutputs",
+ "Confidentiality Reason(s) - Outputs" => "confidentialityOutputsReason",
+ "Consultancy" => "consultancy",
+ "Consultancy Reason(s)" => "consultancyReason",
+ "Amendment Reason" => "amendmentReason",
+ "Name" => "supplierName",
+ "Postal Address" => "supplierAddress",
+ "Town/City" => "supplierCity",
+ "Postcode" => "supplierPostcode",
+ "Country" => "supplierCountry",
+ "ABN Exempt" => "supplierABNExempt",
+ "ABN" => "supplierABN",
+ "Branch" => "contactBranch",
+ "Division" => "contactDivision",
+ "Office Postcode" => "contactPostcode"
+ );
+$cnFields = Array();
+foreach(pq('tr') as $tr) {
+ $tra = dom_to_array($tr);
+ if (is_array($tra['th'])) {
+ $fieldName = trim(getTextFromTHNode($tr));
+ } else {
+ $fieldName = trim(str_replace("/th>","",$tra['th']));
+ }
+ $fieldValue = trim(print_r($tra['td'],true));
+ if ($fieldName == "State/Territory" || $fieldName == "Contact Name"
+ || $fieldName == "Contact Phone" || $fieldName == "Contact Email"
+ ||$fieldName == "Amendments") {
+ // do nothing
+ } else if ($fieldName == "Contract Period") {
+ $contractPeriod = explode("to",$fieldValue);
+ $cnFields["contractStart"] = trim($contractPeriod[0]);
+ $cnFields["contractEnd"] = trim($contractPeriod[1]);
+ } else {
+ $fieldName = $datamapping0711[$fieldName];
+
+ if ($fieldName == "parentCN" || $fieldName == "CNID") {
+ if (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+ $fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
+ $fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
+ } elseif ($fieldName == "description") {
+
+ if (is_array($tra['td'])) $fieldValue = print_r($tra['td']['p'],true);
+
+ } elseif ($fieldName == "value" || $fieldName == "supplierABN") {
+ if (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+ $fieldValue = str_replace(Array("$",","," "), "", $fieldValue);
+ //if (!is_numeric($fieldValue)) $fieldValue = 0;
+ if ($fieldValue == "Exempt") $fieldValue = NULL;
+ } elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
+ $fieldValue = date('Y-m-d H:i:s', strtotime($fieldValue));
+ } elseif (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+