Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard

--- a/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls
+++ b/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls
@@ -145,8 +145,7 @@
 ="Department of Foreign Affairs and Trade"	=""	="CN445119"	14-Nov-11 10:08 AM	 	="published"	01-Sep-11	31-Aug-12	300000.00	"Provision of News and Publicity Services"	="FO010911-CPDPMB0035"	="News and publicity services"	="Open"	="PMC2008 P0019"	="SON183611"	="No"	=""	="No"	=""	="No"	=""	""	="MEDIA MONITORS PTY LTD"	="131 Canberra Avenue"	="Griffith"	="2603"	="Australia"	="No"	="11002533851"	="FMB"	="CMD"	="0221"	

 ="Defence Materiel Organisation"	=""	="CN445120"	14-Nov-11 10:08 AM	 	="published"	25-Oct-11	28-Nov-12	133650.00	" Provision of sheave for recovery assist system "	="DNL62K"	="Marine craft systems and subassemblies"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="Beak Engineering"	=""	="Keysborough"	="3173"	="AUSTRALIA"	="No"	="35122373022"	=""	=""	="2600"	

 ="Department of Foreign Affairs and Trade"	=""	="CN445121"	14-Nov-11 10:08 AM	 	="published"	05-Dec-11	30-Jun-12	28728.70	"Provision of Education and Training Services"	="CN051211-CMDSFB0324"	="Education and Training Services"	="Select"	=""		="No"	=""	="No"	=""	="No"	=""	""	="BERLITZ AUSTRALIA PTY"	="Ground Floor 35 Clarence Street"	="Sydney"	="2000"	="Australia"	="No"	="50007090271"	="FMB"	="CMD"	="0221"	

-="Department of Foreign Affairs and Trade"	=""	="CN445122"	14-Nov-11 10:08 AM	 	="published"	29-Aug-11	05-Sep-11	45669.80	"Provision of Management Advisory Services"	="CN290811-IMDIPB0102"	="Management advisory services"	="Direct"	=""		="No"	=""	="No"	=""	="Yes"	="Need for independent research or assessment"	""	="THE BOSTON CONSULTING GROUP PTY LTD"	="Level 28 Chifley Tower
-2 Chifley Square"	="Sydney"	="2000"	="Australia"	="No"	="70007347131"	="FMB"	="CMD"	="0221"	

+="Department of Foreign Affairs and Trade"	=""	="CN445122"	14-Nov-11 10:08 AM	 	="published"	29-Aug-11	05-Sep-11	45669.80	"Provision of Management Advisory Services"	="CN290811-IMDIPB0102"	="Management advisory services"	="Direct"	=""		="No"	=""	="No"	=""	="Yes"	="Need for independent research or assessment"	""	="THE BOSTON CONSULTING GROUP PTY LTD"	="Level 28 Chifley Tower 2 Chifley Square"	="Sydney"	="2000"	="Australia"	="No"	="70007347131"	="FMB"	="CMD"	="0221"	

 ="Department of Foreign Affairs and Trade"	=""	="CN445123"	14-Nov-11 10:08 AM	 	="published"	18-Oct-11	31-Dec-11	30000.00	"Provision of Security and Personal Safety"	="FO181011-CMDDSB0018"	="Security and personal safety"	="Open"	="DFAT09-DID-15"	="SON336277"	="No"	=""	="No"	=""	="No"	=""	""	="DYNAMIQ PTY LTD"	="Level 5, 33 York Street"	="Sydney"	="2000"	="Australia"	="No"	="38115069335"	="FMB"	="CMD"	="0221"	

 ="Department of Foreign Affairs and Trade"	=""	="CN445124"	14-Nov-11 10:08 AM	 	="published"	06-Oct-11	30-Nov-11	44000.00	"Provision of Information Technology Consultation Services"	="CN061011-CMDFSB0028"	="Information technology consultation services"	="Direct"	=""		="No"	=""	="No"	=""	="Yes"	="Need for independent research or assessment"	""	="PHOENIX MANAGEMENT SERVICES PTY LTD"	="111 Settlers Road"	="Royalla"	="2620"	="Australia"	="No"	="11151887802"	="FMB"	="CMD"	="0221"	

 ="Department of Innovation Industry Science and Research"	="CN316202"	="CN316202-A1"	30-Jul-10 04:52 PM	14-Nov-11 10:10 AM	="published"	01-Jul-10	30-Jun-11	229152.00	" Senior Sharepoint Developer "	="ORDER_DEED-001339-0"	="Temporary personnel services"	="Open"	=""	="SON26578"	="No"	=""	="No"	=""	="No"	=""	"Extend engagement"	="MPM GROUP PTY LTD"	="Office 15, First floor Manuka Court"	="11-17 Bougainville Street, Manuka"	="2603"	="AUSTRALIA"	="No"	="62073795105"	="BUSINESS COLLABORATION BRANCH"	="CORPORATE"	="2600"	

@@ -265,9 +264,7 @@
 ="National Archives of Australia"	="CN445191"	="CN445191-A2"	14-Nov-11 12:54 PM	14-Nov-11 03:17 PM	="published"	15-Sep-11	30-Dec-11	150127.00	" Provision of hardware and software services for multitouch interactive "	="POC00970-PO1100058"	="Software"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	"incorrect value input on amendment No. 1"	="Lightwell Pty Ltd"	="Level 3, 48 Chippen St"	="CHIPPENDALE"	="2008"	="AUSTRALIA"	="No"	="84107866660"	="Corporate Services"	="National Archives"	="2600"	

 ="Department of Foreign Affairs and Trade"	=""	="CN445250"	14-Nov-11 03:22 PM	 	="published"	01-Jul-11	30-Jun-12	798000.00	"Provision of Security and Control Equipment"	="CN010711-CMDDSB0327"	="Security and control equipment"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="SMITHS DETECTION PTY LTD"	="Unit 5, Botany Grove Estate, 14 Baker Street"	="Botany"	="2019"	="Australia"	="No"	="66088868062"	="FMB"	="CMD"	="0221"	

 ="Department of Foreign Affairs and Trade"	=""	="CN445251"	14-Nov-11 03:22 PM	 	="published"	10-Nov-11	30-Jun-12	79500.00	"Provision of Software"	="CN101111-IMDISB0238"	="Software"	="Direct"	=""		="No"	=""	="No"	=""	="Yes"	="Need for specialised or professional skills"	""	="I.T. EXECUTIVE CONSULTING PTY. LIMITED"	="Suite 283, 4 Young Street"	="Neutral Bay"	="2089"	="Australia"	="No"	="24096436001"	="FMB"	="CMD"	="0221"	

-="Department of Foreign Affairs and Trade"	=""	="CN445252"	14-Nov-11 03:22 PM	 	="published"	09-Nov-11	31-Mar-12	55000.00	"Provision of Fleet Management Services"	="CN091111-IMDISB0017"	="Fleet management services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="INFINITE CONSULTING PTY LIMITED"	="Phoenix House
-Ground Floor Unit 8
-86-88 Northbourne Ave"	="Braddon"	="2612"	="Australia"	="No"	="56081954645"	="FMB"	="CMD"	="0221"	

+="Department of Foreign Affairs and Trade"	=""	="CN445252"	14-Nov-11 03:22 PM	 	="published"	09-Nov-11	31-Mar-12	55000.00	"Provision of Fleet Management Services"	="CN091111-IMDISB0017"	="Fleet management services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="INFINITE CONSULTING PTY LIMITED"	="Phoenix House Ground Floor Unit 8 86-88 Northbourne Ave"	="Braddon"	="2612"	="Australia"	="No"	="56081954645"	="FMB"	="CMD"	="0221"	

 ="Australian Federal Police"	="CN412688"	="CN412688-A1"	22-Jul-11 09:05 AM	14-Nov-11 03:25 PM	="published"	01-Jul-11	14-Aug-11	32611.87	" Provision of high level technical support and management services of the UNIX environment "	="1015592"	="Computer services"	="Open"	="RFT 8-2007"	="SON85958"	="No"	=""	="No"	=""	="No"	=""	"Contract terminated and value decreased by $220,546.53"	="Paxus Australia Pty Limited"	=""	="Braddon"	="2601"	="AUSTRALIA"	="No"	="35004609616"	="National Procurement and Contracts"	=""	="2600"	

 ="Australian Federal Police"	="CN113130"	="CN113130-A6"	21-Aug-08 08:18 AM	14-Nov-11 03:33 PM	="published"	01-Jul-08	30-Jun-11	785335.94	" Provision of a senior database administrator "	="1011258"	="Computer services"	="Open"	="RFT 8-2007"	="SON85958"	="No"	=""	="No"	=""	="No"	=""	"Agreement have ceased and value reduced by $18,913.66"	="Paxus Australia Pty Limited"	=""	="Dickson"	="2612"	="AUSTRALIA"	="No"	="35004609616"	="National Procurement and Contracts"	=""	="2600"	

 ="Department of Broadband Communications and the Digital Economy"	=""	="CN445255"	14-Nov-11 03:33 PM	 	="published"	11-Nov-11	31-Jan-12	16500.00	"Legal Services - Constitutional Advice"	="0004601966"	="Legal services"	="Open"	="DCON/10/65"	="SON347233"	="Yes"	="Intellectual property"	="Yes"	="Intellectual property"	="Yes"	="Need for specialised or professional skills"	""	="AUSTRALIAN GOVERNMENT SOLICITOR"	="Locked Bag 7246"	="Canberra Mail Centre"	="2610"	="Australia"	="No"	="69405937639"	=""	="LEGAL Legal Group"	="2603"	


--- a/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls
+++ b/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls
@@ -85,7 +85,7 @@
 ="Department of the Prime Minister and Cabinet"	=""	="CN447929"	24-Nov-11 03:33 PM	 	="published"	01-Aug-11	30-Apr-13	588472.00	" Travel Management Services "	="PMC 2011 TMP"	="Travel facilitation"	="Open"	="FIN09/AMG007"	="SON289574"	="No"	=""	="No"	=""	="No"	=""	""	="Australian OpCo Pty Ltd T/A FCm Travel Solutions"	="Level 2, 545 Queen St"	="Brisbane"	="4000"	="AUSTRALIA"	="No"	="20003279534"	="FMB"	="CSD"	="2600"	

 ="AusAid"	="CN330933"	="CN330933-A1"	14-Sep-10 11:16 AM	22-Nov-11 01:33 PM	="published"	17-May-10	31-Oct-11	235400.00	"MFD Printers - Lease & DocuCare"	="D00209"	="Computer printers"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	"value & duration"	="FUJI XEROX"	=""	="NORTH RYDE"	="2113"	="AUSTRALIA"	="No"	="63000341819"	=""	=""	="2601"	

 ="Australian Transaction Reports and Analysis Centre (AUSTRAC)"	=""	="CN443393"	21-Nov-11 11:13 AM	 	="published"	13-Jul-11	13-Jul-11	14085.50	" Provision of training "	="Sept 2011"	="Education and Training Services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="Object Consulting"	=""	="North Sydney"	="2060"	="AUSTRALIA"	="No"	="55003682693"	=""	=""	="2067"	

-="AusAid"	=""	="CN443720"	22-Nov-11 01:32 PM	 	="published"	07-Oct-11	30-Nov-11	53493.00	" Review of the Australian aid program's food security and rural development portfolio "	="D48944/4"	="Environmental management"	="Open"	=""		="No"	=""	="No"	=""	="Yes"	="Need for specialised or professional skills"	""	="Griffin NRM Pty. Ltd.							Griffin NRM Pty Ltd"	=""	="Garran"	="2605"	="AUSTRALIA"	="No"	="67088471523"	=""	=""	="2601"	

+="AusAid"	=""	="CN443720"	22-Nov-11 01:32 PM	 	="published"	07-Oct-11	30-Nov-11	53493.00	" Review of the Australian aid program's food security and rural development portfolio "	="D48944/4"	="Environmental management"	="Open"	=""		="No"	=""	="No"	=""	="Yes"	="Need for specialised or professional skills"	""	="Griffin NRM Pty. Ltd."	=""	="Garran"	="2605"	="AUSTRALIA"	="No"	="67088471523"	=""	=""	="2601"	

 ="Department of Families, Housing, Community Services and Indigenous Affairs"	="CN395069"	="CN395069-A1"	09-Jun-11 02:12 PM	21-Nov-11 09:53 AM	="published"	08-Jun-11	31-Dec-11	26598.00	" Creative and Design Services "	="45395003"	="Graphic design"	="Open"	="07/T585"		="No"	=""	="No"	=""	="No"	=""	"Decrease of Contract Value"	="David & Kate Pty Ltd t/aDogma"	=""	="East St Kilda"	="3183"	="AUSTRALIA"	="No"	="26613552343"	="Communications and Media"	="FaHCSIA"	="2900"	

 ="Department of Families, Housing, Community Services and Indigenous Affairs"	=""	="CN444996"	21-Nov-11 02:58 PM	 	="published"	01-Dec-11	15-Mar-12	10194.80	" Print Closing the Gap Prime Minister's Report 2012 "	="45405148"	="Printing"	="Open"	="08/T629"	="SON206666"	="No"	=""	="No"	=""	="No"	=""	""	="NEW MILLENNIUM PRINT PTY LTD"	=""	="FYSHWICK"	="2609"	="AUSTRALIA"	="No"	="69089018271"	=""	=""	="2900"	

 ="Department of Families, Housing, Community Services and Indigenous Affairs"	="CN402123"	="CN402123-A1"	14-Jul-11 02:24 PM	24-Nov-11 03:58 PM	="published"	01-Jul-11	31-Mar-12	79568.00	" Human Resource Processes  "	="45396974"	="Audit services"	="Open"	="07/T609"	="SON90044"	="No"	=""	="No"	=""	="No"	=""	"variation to extend timeframe"	="Oakton Services Pty Ltd"	=""	="Kingston"	="2604"	="AUSTRALIA"	="No"	="31100103268"	="Legal & Compliance"	="FaHCSIA"	="2900"	


--- a/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls
+++ b/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls
@@ -587,8 +587,7 @@
 ="Department of Health and Ageing - Therapeutic Goods Administration"	="CN428386"	="CN428386-A1"	09-Sep-11 11:35 AM	29-Feb-12 02:29 PM	="published"	12-Sep-11	20-Dec-11	39562.08	"       Engagement of a Contractor - Events & Communications Officer     "	="TGA2011-0343"	="Temporary personnel services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	"Extension of contract"	="Careers Unlimited Pty Ltd"	="35-37 London Circuit"	="Canberra City"	="2601"	="AUSTRALIA"	="No"	="17075304737"	="Office of Parliamentary & Strategic Support"	="Therapeutic Goods Administration"	="2606"	

 ="Department of Defence"	=""	="CN467784"	29-Feb-12 02:36 PM	 	="published"	28-Feb-12	28-Mar-12	12074.28	" Spare Parts Military Vehicles "	="EB5545"	="Motor vehicles"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="JAGUAR LAND ROVER AUSTRALIA"	=""	="TULLAMARINE"	="3043"	="AUSTRALIA"	="No"	="86004352238"	=""	=""	="2600"	

 ="Department of Foreign Affairs and Trade"	=""	="CN467786"	29-Feb-12 02:39 PM	 	="published"	22-Feb-12	21-Feb-15	149669.30	"Provision of Communications Devices and Accessories"	="CN220212-IMDIPB0047"	="Communications Devices and Accessories"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="VOCALITY PTY LTD"	="4/19 Reliance Drive"	="Tuggerah"	="2259"	="Australia"	="No"	="61103706369"	="FMB"	="CMD"	="0221"	

-="Department of Foreign Affairs and Trade"	=""	="CN467787"	29-Feb-12 02:39 PM	 	="published"	24-Feb-12	06-Mar-12	56000.00	"Provision of Public Relation Services"	="CN240212-SMDMEB0016"	="Public relation services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="AUSTRALIA GULF COUNCIL LIMITED"	="Level 7
-8 Spring Street"	="Sydney"	="2000"	="Australia"	="No"	="99143749786"	="FMB"	="CMD"	="0221"	

+="Department of Foreign Affairs and Trade"	=""	="CN467787"	29-Feb-12 02:39 PM	 	="published"	24-Feb-12	06-Mar-12	56000.00	"Provision of Public Relation Services"	="CN240212-SMDMEB0016"	="Public relation services"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	""	="AUSTRALIA GULF COUNCIL LIMITED"	="Level 7 8 Spring Street"	="Sydney"	="2000"	="Australia"	="No"	="99143749786"	="FMB"	="CMD"	="0221"	

 ="Department of Health and Ageing - Therapeutic Goods Administration"	=""	="CN467788"	29-Feb-12 02:44 PM	 	="published"	17-Feb-10	16-Feb-13	990000.00	"       Guarding and Patrol Services at TGA     "	="TGA2012-0094"	="Security guard services"	="Open"	=""		="No"	=""	="No"	=""	="No"	=""	""	="MSS Security Australia Pty Ltd"	="149-155 Milton Street"	="Ashfield"	="2131"	="AUSTRALIA"	="No"	="29100573966"	="Office of Corporate Services"	="Therapeutic Goods Administration"	="2606"	

 ="Defence Materiel Organisation"	=""	="CN467789"	29-Feb-12 02:47 PM	 	="published"	28-Sep-11	03-Oct-11	50168.70	" Lubricating oil "	="JW1EPX/3073"	="Lubricants and oils and greases and anti corrosives"	="Open"	=""		="No"	=""	="No"	=""	="No"	=""	""	="BP Australia Ltd"	=""	="Docklands"	="3008"	="AUSTRALIA"	="No"	="53004085616"	="JFLA"	="MSD"	="2000"	

 ="Defence Materiel Organisation"	="CN465897"	="CN465897-A1"	21-Feb-12 08:23 AM	29-Feb-12 02:49 PM	="published"	20-Feb-12	14-May-12	15534.20	" Lift Motor Vehicle 5 Tonne Capacity "	="KM0AER"	="Tools and General Machinery"	="Direct"	=""		="No"	=""	="No"	=""	="No"	=""	"Change of Agency Reference ID"	="Garage Equipment"	="17 Canberra Street"	="Hemmant"	="4174"	="AUSTRALIA"	="No"	="75820442568"	="LSSB"	="LSD"	="3006"	


file:b/admin/import.php (new)
--- /dev/null
+++ b/admin/import.php
@@ -1,1 +1,263 @@
-
+<?php
+
+include_once ("../lib/common.inc.php");
+
+$contractNoticeFields = array(
+    "importFile",
+    "agencyName",
+    "parentCN",
+    "CNID",
+    "publishDate",
+    "amendDate",
+    "contractStart",
+    "contractEnd",
+    "value",
+    "description",
+    "agencyID",
+    "category",
+    "procurementMethod",
+    "atmID",
+    "SONID",
+    "confidentialityContract",
+    "confidentialityContractReason",
+    "confidentialityOutputs",
+    "confidentialityOutputsReason",
+    "consultancy",
+    "consultancyReason",
+    "amendmentReason",
+    "supplierName",
+    "supplierAddress",
+    "supplierCity",
+    "supplierPostcode",
+    "supplierCountry",
+    "supplierABNExempt",
+    "supplierABN",
+    "contactBranch",
+    "contactDivision",
+    "contactPostcode"
+);
+$agencyFields = array(
+    "agencyName"
+);
+$supplierFields = array(
+    "supplierName",
+    "supplierAddress",
+    "supplierCity",
+    "supplierPostcode",
+    "supplierCountry",
+    "supplierABNExempt",
+    "supplierABN"
+);
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
+foreach ($contractNoticeFields as $key => $f) {
+    $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$contractNoticeInsertQ.= ");";
+$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+
+$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
+foreach ($supplierFields as $key => $f) {
+    $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$supplierInsertQ.= ");";
+$supplierInsertQ = $conn->prepare($supplierInsertQ);
+
+$agencyInsertQ = 'INSERT INTO agency_nametoabn ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
+foreach ($agencyFields as $key => $f) {
+    $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$agencyInsertQ.= ");";
+$agencyInsertQ = $conn->prepare($agencyInsertQ);
+
+function processFile($fpath, $tablename) {
+    global $conn, $contractNoticeFields, $contractNoticeInsertQ, $supplierFields, $supplierInsertQ,$agencyFields, $agencyInsertQ;
+    $row = 1;
+    $handle = fopen($fpath, "r");
+    //"t" mode string translates windows line breaks to unix
+    $datamapping0711 = array(
+        "Agency" => "agencyName",
+        "Parent CN ID" => "parentCN",
+        "CN ID" => "CNID",
+        "Publish Date" => "publishDate",
+        "Amendment Date" => "amendDate",
+        "Status" => "",
+        "StartDate" => "contractStart",
+        "EndDate" => "contractEnd",
+        "Value" => "value",
+        "Description" => "description",
+        "Agency Ref Id" => "agencyID",
+        "Category" => "category",
+        "Procurement Method" => "procurementMethod",
+        "ATM ID" => "atmID",
+        "SON ID" => "SONID",
+        "Confidentiality - Contract" => "confidentialityContract",
+        "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
+        "Confidentiality - Outputs" => "confidentialityOutputs",
+        "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
+        "Consultancy" => "consultancy",
+        "Consultancy Reason(s)" => "consultancyReason",
+        "Amendment Reason" => "amendmentReason",
+        "Supplier Name" => "supplierName",
+        "Supplier Address" => "supplierAddress",
+        "Supplier City" => "supplierCity",
+        "Supplier Postcode" => "supplierPostcode",
+        "Supplier Country" => "supplierCountry",
+        "Supplier ABNExempt" => "supplierABNExempt",
+        "Supplier ABN" => "supplierABN",
+        "Agency Branch" => "contactBranch",
+        "Agency Divison" => "contactDivision",
+        "Agency Postcode" => "contactPostcode",
+        "" => ""
+    );
+    $headers;
+
+    while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+        $num = count($data);
+        if ($row == 3) {
+            $headers = $data;
+        } elseif ($row > 3) {
+            if ($num > count($datamapping0711)) {
+                die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
+            }
+            $contractNoticeInsert = Array();
+            $supplierInsert = Array();
+            $agencyInsert = Array();
+            $contractNoticeInsert[] = $fpath;
+            $keys = array_keys($datamapping0711);
+            for ($c = 0; $c < $num; $c++) {
+                $data[$c] = trim($data[$c], "=");
+                $data[$c] = trim($data[$c], "\"");
+                if ($tablename == "contractnotice") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) {
+                        if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
+                            $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
+                            if ($data[$c] > 0 && $data[$c] != '0') {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = null;
+                            }
+                        } elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
+                            if ($data[$c] > 0 && $data[$c] != '0') {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = null;
+                            }
+                        } elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
+                            $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
+                        } else {
+                            if (strstr("\" =", $data[$c] > 0)) {
+                                die("Invalid Description field" . $contractNoticeInsert);
+                            }
+                            $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));
+
+                            $contractNoticeInsert[] = $colvalue;
+                        }
+                    }
+                } else if ($tablename == "supplierdetails") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) {
+                        if (($datamapping0711[$headers[$c]]) == "supplierABN") {
+                            if ($data[$c] > 0) {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = 0;
+                            }
+                        } else {
+                            $supplierInsert[] = $data[$c];
+                        }
+                    }
+                } else if ($tablename == "agency") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) {
+                        $agencyInsert[] = $data[$c];
+                    }
+                }
+            }
+            flush();
+            if ($tablename == "contractnotice") {
+                $contractNoticeInsertQ->execute($contractNoticeInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    foreach ($contractNoticeFields as $key => $cnf) {
+                        echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
+                    }
+                    echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+                }
+            } else if ($tablename == "supplierdetails") {
+                $supplierInsertQ->execute($supplierInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
+                }
+            } else if ($tablename == "agency") {
+                $agencyInsertQ->execute($agencyInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
+                }
+            }
+            flush();
+            //echo "<hr>\n";
+        }
+        $row++;
+    }
+    fclose($handle);
+    $contractNoticeInsertQ->closeCursor();
+    $supplierInsertQ->closeCursor();
+    $agencyInsertQ->closeCursor();
+ 
+    return $success;
+}
+
+$path = 'data/';
+if ($_REQUEST["fname"] == "") {
+    echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+    $dhandle = opendir($path);
+    // define an array to hold the files
+    $files = array();
+    if ($dhandle) {
+        // loop through all of the files
+        while (false !== ($fname = readdir($dhandle))) {
+            if (($fname != '.') && ($fname != '..')) {
+                $files[date("c", filemtime($path . $fname)).md5($fname)] = $fname;
+                  }
+        }
+    }
+    ksort($files);
+    foreach ($files as $date => $fname) {
+                  echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . $date . "<br/>";
+
+    }
+} else {
+    $success = 0;
+    $fname = $_REQUEST["fname"];
+    echo " ============== $fname  ============== <br>";
+    flush();
+    $success+= processFile($path . $fname, "contractnotice");
+    $success+= processFile($path . $fname, "agency");
+    $success+= processFile($path . $fname, "supplier");
+    echo "<br> $success records successfully created";
+    
+    flush();
+       // run post import data processing
+// cn
+    echo "link amend<br>";
+    include ("linkAmendments.php");
+    echo "update UNSPSC<br>";
+    include ("updateUNSPSC.php");
+// agency
+//include ("setAgencyStatus.php");
+//include ("setAgencyURLABN.php");
+}
+?>
+

--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -1,5 +1,11 @@
 <?php
 include_once ("../lib/common.inc.php");
+$query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\'';
+$query = $conn->prepare($query);
+	$query->execute();
+        $query = 'update contractnotice set "supplierABN" = null where "supplierABN" = \'0\'';
+$query = $conn->prepare($query);
+	$query->execute();
 $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c  LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
 WHERE
 c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
@@ -7,6 +13,7 @@
 	$query->execute();
 		databaseError($conn->errorInfo());
 foreach ($query->fetchAll() as $row) {
+    if ($row['parentCN'] != 0) {
   $conn->exec('UPDATE contractnotice SET "childCN" = \'' .
 			 $row['CNID'] . '\' where "CNID" = \'' .
 			 $row['parentCN'] . '\';');
@@ -19,7 +26,7 @@
                   echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
   "<br>\n";
   else print_r($errors);
-
+    }
 }
 // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
 $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN

--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -7,7 +7,7 @@
   	<dependency>
   		<groupId>org.neo4j</groupId>
   		<artifactId>neo4j</artifactId>
-  		<version>1.5</version>
+  		<version>1.6.1</version>
   	</dependency>
   	<dependency>
     <groupId>postgresql</groupId>

--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -88,26 +88,26 @@
                     + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
                     + "  contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
             String previousAgency = "";
-GraphDatabaseService gds = inserter.getGraphDbService(); 
-HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
-HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
-        
+            GraphDatabaseService gds = inserter.getGraphDbService();
+            HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
+            HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
+
             // Loop through the result set
             while (rs.next()) {
                 long supplierID, agencyID;
                 String supplierKey;
-                    if (agencyIDs.get(rs.getString("agencyName")) == null) {
-                       Node myNode = gds.createNode(); 
-                        myNode.setProperty("Label", rs.getString("agencyName"));
-                          myNode.setProperty("type", "agency");
-                        agencyIDs.put(rs.getString("agencyName"), myNode.getId());
-                        if (myNode.getId() %100 == 0) {
-    System.out.println("Agency "+myNode.getId());
-}
-                    } 
-                       agencyID = agencyIDs.get(rs.getString("agencyName"));
-                
-                
+                if (agencyIDs.get(rs.getString("agencyName")) == null) {
+                    Node myNode = gds.createNode();
+                    myNode.setProperty("Label", rs.getString("agencyName"));
+                    myNode.setProperty("type", "agency");
+                    agencyIDs.put(rs.getString("agencyName"), myNode.getId());
+                    if (myNode.getId() % 100 == 0) {
+                        System.out.println("Agency " + myNode.getId());
+                    }
+                }
+                agencyID = agencyIDs.get(rs.getString("agencyName"));
+
+
                 if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
                     supplierKey = rs.getString("supplierABN");
                 } else {
@@ -115,19 +115,19 @@
                 }
                 // inject some data 
                 if (supplierIDs.get(supplierKey) == null) {
-                    Node myNode = gds.createNode(); 
-                        myNode.setProperty("Label", rs.getString("supplierName"));
-                            myNode.setProperty("type", "supplier");
+                    Node myNode = gds.createNode();
+                    myNode.setProperty("Label", rs.getString("supplierName"));
+                    myNode.setProperty("type", "supplier");
                     supplierIDs.put(supplierKey, myNode.getId());
-                    if (myNode.getId() %1000 == 0) {
-    System.out.println("Supplier "+myNode.getId());
-}
+                    if (myNode.getId() % 1000 == 0) {
+                        System.out.println("Supplier " + myNode.getId());
+                    }
                 }
                 supplierID = supplierIDs.get(supplierKey);
 
 
                 long rel = inserter.createRelationship(agencyID, supplierID,
-                       DynamicRelationshipType.withName("KNOWS"), null);
+                        DynamicRelationshipType.withName("KNOWS"), null);
                 inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
 
             }
@@ -154,6 +154,4 @@
         indexProvider.shutdown();
         inserter.shutdown();
     }
-
-   
 }