Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
--- a/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls
+++ b/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls
@@ -145,8 +145,7 @@
="Department of Foreign Affairs and Trade" ="" ="CN445119" 14-Nov-11 10:08 AM ="published" 01-Sep-11 31-Aug-12 300000.00 "Provision of News and Publicity Services" ="FO010911-CPDPMB0035" ="News and publicity services" ="Open" ="PMC2008 P0019" ="SON183611" ="No" ="" ="No" ="" ="No" ="" "" ="MEDIA MONITORS PTY LTD" ="131 Canberra Avenue" ="Griffith" ="2603" ="Australia" ="No" ="11002533851" ="FMB" ="CMD" ="0221"
="Defence Materiel Organisation" ="" ="CN445120" 14-Nov-11 10:08 AM ="published" 25-Oct-11 28-Nov-12 133650.00 " Provision of sheave for recovery assist system " ="DNL62K" ="Marine craft systems and subassemblies" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Beak Engineering" ="" ="Keysborough" ="3173" ="AUSTRALIA" ="No" ="35122373022" ="" ="" ="2600"
="Department of Foreign Affairs and Trade" ="" ="CN445121" 14-Nov-11 10:08 AM ="published" 05-Dec-11 30-Jun-12 28728.70 "Provision of Education and Training Services" ="CN051211-CMDSFB0324" ="Education and Training Services" ="Select" ="" ="No" ="" ="No" ="" ="No" ="" "" ="BERLITZ AUSTRALIA PTY" ="Ground Floor 35 Clarence Street" ="Sydney" ="2000" ="Australia" ="No" ="50007090271" ="FMB" ="CMD" ="0221"
-="Department of Foreign Affairs and Trade" ="" ="CN445122" 14-Nov-11 10:08 AM ="published" 29-Aug-11 05-Sep-11 45669.80 "Provision of Management Advisory Services" ="CN290811-IMDIPB0102" ="Management advisory services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="THE BOSTON CONSULTING GROUP PTY LTD" ="Level 28 Chifley Tower
-2 Chifley Square" ="Sydney" ="2000" ="Australia" ="No" ="70007347131" ="FMB" ="CMD" ="0221"
+="Department of Foreign Affairs and Trade" ="" ="CN445122" 14-Nov-11 10:08 AM ="published" 29-Aug-11 05-Sep-11 45669.80 "Provision of Management Advisory Services" ="CN290811-IMDIPB0102" ="Management advisory services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="THE BOSTON CONSULTING GROUP PTY LTD" ="Level 28 Chifley Tower 2 Chifley Square" ="Sydney" ="2000" ="Australia" ="No" ="70007347131" ="FMB" ="CMD" ="0221"
="Department of Foreign Affairs and Trade" ="" ="CN445123" 14-Nov-11 10:08 AM ="published" 18-Oct-11 31-Dec-11 30000.00 "Provision of Security and Personal Safety" ="FO181011-CMDDSB0018" ="Security and personal safety" ="Open" ="DFAT09-DID-15" ="SON336277" ="No" ="" ="No" ="" ="No" ="" "" ="DYNAMIQ PTY LTD" ="Level 5, 33 York Street" ="Sydney" ="2000" ="Australia" ="No" ="38115069335" ="FMB" ="CMD" ="0221"
="Department of Foreign Affairs and Trade" ="" ="CN445124" 14-Nov-11 10:08 AM ="published" 06-Oct-11 30-Nov-11 44000.00 "Provision of Information Technology Consultation Services" ="CN061011-CMDFSB0028" ="Information technology consultation services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="PHOENIX MANAGEMENT SERVICES PTY LTD" ="111 Settlers Road" ="Royalla" ="2620" ="Australia" ="No" ="11151887802" ="FMB" ="CMD" ="0221"
="Department of Innovation Industry Science and Research" ="CN316202" ="CN316202-A1" 30-Jul-10 04:52 PM 14-Nov-11 10:10 AM ="published" 01-Jul-10 30-Jun-11 229152.00 " Senior Sharepoint Developer " ="ORDER_DEED-001339-0" ="Temporary personnel services" ="Open" ="" ="SON26578" ="No" ="" ="No" ="" ="No" ="" "Extend engagement" ="MPM GROUP PTY LTD" ="Office 15, First floor Manuka Court" ="11-17 Bougainville Street, Manuka" ="2603" ="AUSTRALIA" ="No" ="62073795105" ="BUSINESS COLLABORATION BRANCH" ="CORPORATE" ="2600"
@@ -265,9 +264,7 @@
="National Archives of Australia" ="CN445191" ="CN445191-A2" 14-Nov-11 12:54 PM 14-Nov-11 03:17 PM ="published" 15-Sep-11 30-Dec-11 150127.00 " Provision of hardware and software services for multitouch interactive " ="POC00970-PO1100058" ="Software" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "incorrect value input on amendment No. 1" ="Lightwell Pty Ltd" ="Level 3, 48 Chippen St" ="CHIPPENDALE" ="2008" ="AUSTRALIA" ="No" ="84107866660" ="Corporate Services" ="National Archives" ="2600"
="Department of Foreign Affairs and Trade" ="" ="CN445250" 14-Nov-11 03:22 PM ="published" 01-Jul-11 30-Jun-12 798000.00 "Provision of Security and Control Equipment" ="CN010711-CMDDSB0327" ="Security and control equipment" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="SMITHS DETECTION PTY LTD" ="Unit 5, Botany Grove Estate, 14 Baker Street" ="Botany" ="2019" ="Australia" ="No" ="66088868062" ="FMB" ="CMD" ="0221"
="Department of Foreign Affairs and Trade" ="" ="CN445251" 14-Nov-11 03:22 PM ="published" 10-Nov-11 30-Jun-12 79500.00 "Provision of Software" ="CN101111-IMDISB0238" ="Software" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="I.T. EXECUTIVE CONSULTING PTY. LIMITED" ="Suite 283, 4 Young Street" ="Neutral Bay" ="2089" ="Australia" ="No" ="24096436001" ="FMB" ="CMD" ="0221"
-="Department of Foreign Affairs and Trade" ="" ="CN445252" 14-Nov-11 03:22 PM ="published" 09-Nov-11 31-Mar-12 55000.00 "Provision of Fleet Management Services" ="CN091111-IMDISB0017" ="Fleet management services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="INFINITE CONSULTING PTY LIMITED" ="Phoenix House
-Ground Floor Unit 8
-86-88 Northbourne Ave" ="Braddon" ="2612" ="Australia" ="No" ="56081954645" ="FMB" ="CMD" ="0221"
+="Department of Foreign Affairs and Trade" ="" ="CN445252" 14-Nov-11 03:22 PM ="published" 09-Nov-11 31-Mar-12 55000.00 "Provision of Fleet Management Services" ="CN091111-IMDISB0017" ="Fleet management services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="INFINITE CONSULTING PTY LIMITED" ="Phoenix House Ground Floor Unit 8 86-88 Northbourne Ave" ="Braddon" ="2612" ="Australia" ="No" ="56081954645" ="FMB" ="CMD" ="0221"
="Australian Federal Police" ="CN412688" ="CN412688-A1" 22-Jul-11 09:05 AM 14-Nov-11 03:25 PM ="published" 01-Jul-11 14-Aug-11 32611.87 " Provision of high level technical support and management services of the UNIX environment " ="1015592" ="Computer services" ="Open" ="RFT 8-2007" ="SON85958" ="No" ="" ="No" ="" ="No" ="" "Contract terminated and value decreased by $220,546.53" ="Paxus Australia Pty Limited" ="" ="Braddon" ="2601" ="AUSTRALIA" ="No" ="35004609616" ="National Procurement and Contracts" ="" ="2600"
="Australian Federal Police" ="CN113130" ="CN113130-A6" 21-Aug-08 08:18 AM 14-Nov-11 03:33 PM ="published" 01-Jul-08 30-Jun-11 785335.94 " Provision of a senior database administrator " ="1011258" ="Computer services" ="Open" ="RFT 8-2007" ="SON85958" ="No" ="" ="No" ="" ="No" ="" "Agreement have ceased and value reduced by $18,913.66" ="Paxus Australia Pty Limited" ="" ="Dickson" ="2612" ="AUSTRALIA" ="No" ="35004609616" ="National Procurement and Contracts" ="" ="2600"
="Department of Broadband Communications and the Digital Economy" ="" ="CN445255" 14-Nov-11 03:33 PM ="published" 11-Nov-11 31-Jan-12 16500.00 "Legal Services - Constitutional Advice" ="0004601966" ="Legal services" ="Open" ="DCON/10/65" ="SON347233" ="Yes" ="Intellectual property" ="Yes" ="Intellectual property" ="Yes" ="Need for specialised or professional skills" "" ="AUSTRALIAN GOVERNMENT SOLICITOR" ="Locked Bag 7246" ="Canberra Mail Centre" ="2610" ="Australia" ="No" ="69405937639" ="" ="LEGAL Legal Group" ="2603"
--- a/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls
+++ b/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls
@@ -85,7 +85,7 @@
="Department of the Prime Minister and Cabinet" ="" ="CN447929" 24-Nov-11 03:33 PM ="published" 01-Aug-11 30-Apr-13 588472.00 " Travel Management Services " ="PMC 2011 TMP" ="Travel facilitation" ="Open" ="FIN09/AMG007" ="SON289574" ="No" ="" ="No" ="" ="No" ="" "" ="Australian OpCo Pty Ltd T/A FCm Travel Solutions" ="Level 2, 545 Queen St" ="Brisbane" ="4000" ="AUSTRALIA" ="No" ="20003279534" ="FMB" ="CSD" ="2600"
="AusAid" ="CN330933" ="CN330933-A1" 14-Sep-10 11:16 AM 22-Nov-11 01:33 PM ="published" 17-May-10 31-Oct-11 235400.00 "MFD Printers - Lease & DocuCare" ="D00209" ="Computer printers" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "value & duration" ="FUJI XEROX" ="" ="NORTH RYDE" ="2113" ="AUSTRALIA" ="No" ="63000341819" ="" ="" ="2601"
="Australian Transaction Reports and Analysis Centre (AUSTRAC)" ="" ="CN443393" 21-Nov-11 11:13 AM ="published" 13-Jul-11 13-Jul-11 14085.50 " Provision of training " ="Sept 2011" ="Education and Training Services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Object Consulting" ="" ="North Sydney" ="2060" ="AUSTRALIA" ="No" ="55003682693" ="" ="" ="2067"
-="AusAid" ="" ="CN443720" 22-Nov-11 01:32 PM ="published" 07-Oct-11 30-Nov-11 53493.00 " Review of the Australian aid program's food security and rural development portfolio " ="D48944/4" ="Environmental management" ="Open" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="Griffin NRM Pty. Ltd. Griffin NRM Pty Ltd" ="" ="Garran" ="2605" ="AUSTRALIA" ="No" ="67088471523" ="" ="" ="2601"
+="AusAid" ="" ="CN443720" 22-Nov-11 01:32 PM ="published" 07-Oct-11 30-Nov-11 53493.00 " Review of the Australian aid program's food security and rural development portfolio " ="D48944/4" ="Environmental management" ="Open" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="Griffin NRM Pty. Ltd." ="" ="Garran" ="2605" ="AUSTRALIA" ="No" ="67088471523" ="" ="" ="2601"
="Department of Families, Housing, Community Services and Indigenous Affairs" ="CN395069" ="CN395069-A1" 09-Jun-11 02:12 PM 21-Nov-11 09:53 AM ="published" 08-Jun-11 31-Dec-11 26598.00 " Creative and Design Services " ="45395003" ="Graphic design" ="Open" ="07/T585" ="No" ="" ="No" ="" ="No" ="" "Decrease of Contract Value" ="David & Kate Pty Ltd t/aDogma" ="" ="East St Kilda" ="3183" ="AUSTRALIA" ="No" ="26613552343" ="Communications and Media" ="FaHCSIA" ="2900"
="Department of Families, Housing, Community Services and Indigenous Affairs" ="" ="CN444996" 21-Nov-11 02:58 PM ="published" 01-Dec-11 15-Mar-12 10194.80 " Print Closing the Gap Prime Minister's Report 2012 " ="45405148" ="Printing" ="Open" ="08/T629" ="SON206666" ="No" ="" ="No" ="" ="No" ="" "" ="NEW MILLENNIUM PRINT PTY LTD" ="" ="FYSHWICK" ="2609" ="AUSTRALIA" ="No" ="69089018271" ="" ="" ="2900"
="Department of Families, Housing, Community Services and Indigenous Affairs" ="CN402123" ="CN402123-A1" 14-Jul-11 02:24 PM 24-Nov-11 03:58 PM ="published" 01-Jul-11 31-Mar-12 79568.00 " Human Resource Processes " ="45396974" ="Audit services" ="Open" ="07/T609" ="SON90044" ="No" ="" ="No" ="" ="No" ="" "variation to extend timeframe" ="Oakton Services Pty Ltd" ="" ="Kingston" ="2604" ="AUSTRALIA" ="No" ="31100103268" ="Legal & Compliance" ="FaHCSIA" ="2900"
--- a/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls
+++ b/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls
@@ -587,8 +587,7 @@
="Department of Health and Ageing - Therapeutic Goods Administration" ="CN428386" ="CN428386-A1" 09-Sep-11 11:35 AM 29-Feb-12 02:29 PM ="published" 12-Sep-11 20-Dec-11 39562.08 " Engagement of a Contractor - Events & Communications Officer " ="TGA2011-0343" ="Temporary personnel services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "Extension of contract" ="Careers Unlimited Pty Ltd" ="35-37 London Circuit" ="Canberra City" ="2601" ="AUSTRALIA" ="No" ="17075304737" ="Office of Parliamentary & Strategic Support" ="Therapeutic Goods Administration" ="2606"
="Department of Defence" ="" ="CN467784" 29-Feb-12 02:36 PM ="published" 28-Feb-12 28-Mar-12 12074.28 " Spare Parts Military Vehicles " ="EB5545" ="Motor vehicles" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="JAGUAR LAND ROVER AUSTRALIA" ="" ="TULLAMARINE" ="3043" ="AUSTRALIA" ="No" ="86004352238" ="" ="" ="2600"
="Department of Foreign Affairs and Trade" ="" ="CN467786" 29-Feb-12 02:39 PM ="published" 22-Feb-12 21-Feb-15 149669.30 "Provision of Communications Devices and Accessories" ="CN220212-IMDIPB0047" ="Communications Devices and Accessories" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="VOCALITY PTY LTD" ="4/19 Reliance Drive" ="Tuggerah" ="2259" ="Australia" ="No" ="61103706369" ="FMB" ="CMD" ="0221"
-="Department of Foreign Affairs and Trade" ="" ="CN467787" 29-Feb-12 02:39 PM ="published" 24-Feb-12 06-Mar-12 56000.00 "Provision of Public Relation Services" ="CN240212-SMDMEB0016" ="Public relation services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="AUSTRALIA GULF COUNCIL LIMITED" ="Level 7
-8 Spring Street" ="Sydney" ="2000" ="Australia" ="No" ="99143749786" ="FMB" ="CMD" ="0221"
+="Department of Foreign Affairs and Trade" ="" ="CN467787" 29-Feb-12 02:39 PM ="published" 24-Feb-12 06-Mar-12 56000.00 "Provision of Public Relation Services" ="CN240212-SMDMEB0016" ="Public relation services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="AUSTRALIA GULF COUNCIL LIMITED" ="Level 7 8 Spring Street" ="Sydney" ="2000" ="Australia" ="No" ="99143749786" ="FMB" ="CMD" ="0221"
="Department of Health and Ageing - Therapeutic Goods Administration" ="" ="CN467788" 29-Feb-12 02:44 PM ="published" 17-Feb-10 16-Feb-13 990000.00 " Guarding and Patrol Services at TGA " ="TGA2012-0094" ="Security guard services" ="Open" ="" ="No" ="" ="No" ="" ="No" ="" "" ="MSS Security Australia Pty Ltd" ="149-155 Milton Street" ="Ashfield" ="2131" ="AUSTRALIA" ="No" ="29100573966" ="Office of Corporate Services" ="Therapeutic Goods Administration" ="2606"
="Defence Materiel Organisation" ="" ="CN467789" 29-Feb-12 02:47 PM ="published" 28-Sep-11 03-Oct-11 50168.70 " Lubricating oil " ="JW1EPX/3073" ="Lubricants and oils and greases and anti corrosives" ="Open" ="" ="No" ="" ="No" ="" ="No" ="" "" ="BP Australia Ltd" ="" ="Docklands" ="3008" ="AUSTRALIA" ="No" ="53004085616" ="JFLA" ="MSD" ="2000"
="Defence Materiel Organisation" ="CN465897" ="CN465897-A1" 21-Feb-12 08:23 AM 29-Feb-12 02:49 PM ="published" 20-Feb-12 14-May-12 15534.20 " Lift Motor Vehicle 5 Tonne Capacity " ="KM0AER" ="Tools and General Machinery" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "Change of Agency Reference ID" ="Garage Equipment" ="17 Canberra Street" ="Hemmant" ="4174" ="AUSTRALIA" ="No" ="75820442568" ="LSSB" ="LSD" ="3006"
--- /dev/null
+++ b/admin/import.php
@@ -1,1 +1,263 @@
-
+<?php
+
+include_once ("../lib/common.inc.php");
+
+$contractNoticeFields = array(
+ "importFile",
+ "agencyName",
+ "parentCN",
+ "CNID",
+ "publishDate",
+ "amendDate",
+ "contractStart",
+ "contractEnd",
+ "value",
+ "description",
+ "agencyID",
+ "category",
+ "procurementMethod",
+ "atmID",
+ "SONID",
+ "confidentialityContract",
+ "confidentialityContractReason",
+ "confidentialityOutputs",
+ "confidentialityOutputsReason",
+ "consultancy",
+ "consultancyReason",
+ "amendmentReason",
+ "supplierName",
+ "supplierAddress",
+ "supplierCity",
+ "supplierPostcode",
+ "supplierCountry",
+ "supplierABNExempt",
+ "supplierABN",
+ "contactBranch",
+ "contactDivision",
+ "contactPostcode"
+);
+$agencyFields = array(
+ "agencyName"
+);
+$supplierFields = array(
+ "supplierName",
+ "supplierAddress",
+ "supplierCity",
+ "supplierPostcode",
+ "supplierCountry",
+ "supplierABNExempt",
+ "supplierABN"
+);
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
+foreach ($contractNoticeFields as $key => $f) {
+ $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$contractNoticeInsertQ.= ");";
+$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+
+$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
+foreach ($supplierFields as $key => $f) {
+ $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$supplierInsertQ.= ");";
+$supplierInsertQ = $conn->prepare($supplierInsertQ);
+
+$agencyInsertQ = 'INSERT INTO agency_nametoabn ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
+foreach ($agencyFields as $key => $f) {
+ $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$agencyInsertQ.= ");";
+$agencyInsertQ = $conn->prepare($agencyInsertQ);
+
+function processFile($fpath, $tablename) {
+ global $conn, $contractNoticeFields, $contractNoticeInsertQ, $supplierFields, $supplierInsertQ,$agencyFields, $agencyInsertQ;
+ $row = 1;
+ $handle = fopen($fpath, "r");
+ //"t" mode string translates windows line breaks to unix
+ $datamapping0711 = array(
+ "Agency" => "agencyName",
+ "Parent CN ID" => "parentCN",
+ "CN ID" => "CNID",
+ "Publish Date" => "publishDate",
+ "Amendment Date" => "amendDate",
+ "Status" => "",
+ "StartDate" => "contractStart",
+ "EndDate" => "contractEnd",
+ "Value" => "value",
+ "Description" => "description",
+ "Agency Ref Id" => "agencyID",
+ "Category" => "category",
+ "Procurement Method" => "procurementMethod",
+ "ATM ID" => "atmID",
+ "SON ID" => "SONID",
+ "Confidentiality - Contract" => "confidentialityContract",
+ "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
+ "Confidentiality - Outputs" => "confidentialityOutputs",
+ "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
+ "Consultancy" => "consultancy",
+ "Consultancy Reason(s)" => "consultancyReason",
+ "Amendment Reason" => "amendmentReason",
+ "Supplier Name" => "supplierName",
+ "Supplier Address" => "supplierAddress",
+ "Supplier City" => "supplierCity",
+ "Supplier Postcode" => "supplierPostcode",
+ "Supplier Country" => "supplierCountry",
+ "Supplier ABNExempt" => "supplierABNExempt",
+ "Supplier ABN" => "supplierABN",
+ "Agency Branch" => "contactBranch",
+ "Agency Divison" => "contactDivision",
+ "Agency Postcode" => "contactPostcode",
+ "" => ""
+ );
+ $headers;
+
+ while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+ $num = count($data);
+ if ($row == 3) {
+ $headers = $data;
+ } elseif ($row > 3) {
+ if ($num > count($datamapping0711)) {
+ die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
+ }
+ $contractNoticeInsert = Array();
+ $supplierInsert = Array();
+ $agencyInsert = Array();
+ $contractNoticeInsert[] = $fpath;
+ $keys = array_keys($datamapping0711);
+ for ($c = 0; $c < $num; $c++) {
+ $data[$c] = trim($data[$c], "=");
+ $data[$c] = trim($data[$c], "\"");
+ if ($tablename == "contractnotice") {
+ if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) {
+ if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
+ $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
+ if ($data[$c] > 0 && $data[$c] != '0') {
+ $contractNoticeInsert[] = $data[$c];
+ } else {
+ $contractNoticeInsert[] = null;
+ }
+ } elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
+ if ($data[$c] > 0 && $data[$c] != '0') {
+ $contractNoticeInsert[] = $data[$c];
+ } else {
+ $contractNoticeInsert[] = null;
+ }
+ } elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
+ $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
+ } else {
+ if (strstr("\" =", $data[$c] > 0)) {
+ die("Invalid Description field" . $contractNoticeInsert);
+ }
+ $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));
+
+ $contractNoticeInsert[] = $colvalue;
+ }
+ }
+ } else if ($tablename == "supplierdetails") {
+ if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) {
+ if (($datamapping0711[$headers[$c]]) == "supplierABN") {
+ if ($data[$c] > 0) {
+ $contractNoticeInsert[] = $data[$c];
+ } else {
+ $contractNoticeInsert[] = 0;
+ }
+ } else {
+ $supplierInsert[] = $data[$c];
+ }
+ }
+ } else if ($tablename == "agency") {
+ if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) {
+ $agencyInsert[] = $data[$c];
+ }
+ }
+ }
+ flush();
+ if ($tablename == "contractnotice") {
+ $contractNoticeInsertQ->execute($contractNoticeInsert);
+ $errors = $conn->errorInfo();
+ if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+
+ } elseif ($errors[1] == 0) {
+ $success++;
+ } else {
+ foreach ($contractNoticeFields as $key => $cnf) {
+ echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
+ }
+ echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+ }
+ } else if ($tablename == "supplierdetails") {
+ $supplierInsertQ->execute($supplierInsert);
+ $errors = $conn->errorInfo();
+ if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+
+ } elseif ($errors[1] == 0) {
+ $success++;
+ } else {
+ echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
+ }
+ } else if ($tablename == "agency") {
+ $agencyInsertQ->execute($agencyInsert);
+ $errors = $conn->errorInfo();
+ if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+
+ } elseif ($errors[1] == 0) {
+ $success++;
+ } else {
+ echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
+ }
+ }
+ flush();
+ //echo "<hr>\n";
+ }
+ $row++;
+ }
+ fclose($handle);
+ $contractNoticeInsertQ->closeCursor();
+ $supplierInsertQ->closeCursor();
+ $agencyInsertQ->closeCursor();
+
+ return $success;
+}
+
+$path = 'data/';
+if ($_REQUEST["fname"] == "") {
+ echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+ $dhandle = opendir($path);
+ // define an array to hold the files
+ $files = array();
+ if ($dhandle) {
+ // loop through all of the files
+ while (false !== ($fname = readdir($dhandle))) {
+ if (($fname != '.') && ($fname != '..')) {
+ $files[date("c", filemtime($path . $fname)).md5($fname)] = $fname;
+ }
+ }
+ }
+ ksort($files);
+ foreach ($files as $date => $fname) {
+ echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
+
+ }
+} else {
+ $success = 0;
+ $fname = $_REQUEST["fname"];
+ echo " ============== $fname ============== <br>";
+ flush();
+ $success+= processFile($path . $fname, "contractnotice");
+ $success+= processFile($path . $fname, "agency");
+ $success+= processFile($path . $fname, "supplier");
+ echo "<br> $success records successfully created";
+
+ flush();
+ // run post import data processing
+// cn
+ echo "link amend<br>";
+ include ("linkAmendments.php");
+ echo "update UNSPSC<br>";
+ include ("updateUNSPSC.php");
+// agency
+//include ("setAgencyStatus.php");
+//include ("setAgencyURLABN.php");
+}
+?>
+
--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -1,5 +1,11 @@
<?php
include_once ("../lib/common.inc.php");
+$query = 'update contractnotice set "parentCN" = null where "parentCN" = \'0\'';
+$query = $conn->prepare($query);
+ $query->execute();
+ $query = 'update contractnotice set "supplierABN" = null where "supplierABN" = \'0\'';
+$query = $conn->prepare($query);
+ $query->execute();
$query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
WHERE
c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
@@ -7,6 +13,7 @@
$query->execute();
databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) {
+ if ($row['parentCN'] != 0) {
$conn->exec('UPDATE contractnotice SET "childCN" = \'' .
$row['CNID'] . '\' where "CNID" = \'' .
$row['parentCN'] . '\';');
@@ -19,7 +26,7 @@
echo $row['CNID'] . " linked to parent " . $row['parentCN'] .
"<br>\n";
else print_r($errors);
-
+ }
}
// also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0
$query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN
--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -7,7 +7,7 @@
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId>
- <version>1.5</version>
+ <version>1.6.1</version>
</dependency>
<dependency>
<groupId>postgresql</groupId>
--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -88,26 +88,26 @@
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
String previousAgency = "";
-GraphDatabaseService gds = inserter.getGraphDbService();
-HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
-HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
-
+ GraphDatabaseService gds = inserter.getGraphDbService();
+ HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
+ HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
+
// Loop through the result set
while (rs.next()) {
long supplierID, agencyID;
String supplierKey;
- if (agencyIDs.get(rs.getString("agencyName")) == null) {
- Node myNode = gds.createNode();
- myNode.setProperty("Label", rs.getString("agencyName"));
- myNode.setProperty("type", "agency");
- agencyIDs.put(rs.getString("agencyName"), myNode.getId());
- if (myNode.getId() %100 == 0) {
- System.out.println("Agency "+myNode.getId());
-}
- }
- agencyID = agencyIDs.get(rs.getString("agencyName"));
-
-
+ if (agencyIDs.get(rs.getString("agencyName")) == null) {
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("agencyName"));
+ myNode.setProperty("type", "agency");
+ agencyIDs.put(rs.getString("agencyName"), myNode.getId());
+ if (myNode.getId() % 100 == 0) {
+ System.out.println("Agency " + myNode.getId());
+ }
+ }
+ agencyID = agencyIDs.get(rs.getString("agencyName"));
+
+
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
supplierKey = rs.getString("supplierABN");
} else {
@@ -115,19 +115,19 @@
}
// inject some data
if (supplierIDs.get(supplierKey) == null) {
- Node myNode = gds.createNode();
- myNode.setProperty("Label", rs.getString("supplierName"));
- myNode.setProperty("type", "supplier");
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("supplierName"));
+ myNode.setProperty("type", "supplier");
supplierIDs.put(supplierKey, myNode.getId());
- if (myNode.getId() %1000 == 0) {
- System.out.println("Supplier "+myNode.getId());
-}
+ if (myNode.getId() % 1000 == 0) {
+ System.out.println("Supplier " + myNode.getId());
+ }
}
supplierID = supplierIDs.get(supplierKey);
long rel = inserter.createRelationship(agencyID, supplierID,
- DynamicRelationshipType.withName("KNOWS"), null);
+ DynamicRelationshipType.withName("KNOWS"), null);
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
}
@@ -154,6 +154,4 @@
indexProvider.shutdown();
inserter.shutdown();
}
-
-
}
Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
--- a/admin/updateReport.php
+++ b/admin/updateReport.php
@@ -1,24 +1,24 @@
<?php
include_once('../lib/common.inc.php');
-$query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
+$query = 'SELECT max(DATE("importDate")) as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
FROM contractnotice WHERE "childCN" is null AND "parentCN" is null AND DATE("importDate") in (select * from (SELECT DATE("importDate")
FROM contractnotice ORDER BY "importDate" DESC limit 1) alias)';
$result = $conn->query($query);
$stats = $result->fetch();
echo $stats['count'] . ' new records in period ' . $stats['start'] . ' to ' . $stats['end'] .'<br>';
-$query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
+$query = 'SELECT max(DATE("importDate")) as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end
FROM contractnotice WHERE ("childCN" is not null OR "parentCN" is not null) AND DATE("importDate") in (select * from (SELECT DATE("importDate")
-FROM contractnotice ORDER BY importDate DESC limit 1) alias)';
+FROM contractnotice ORDER BY "importDate" DESC limit 1) alias)';
$result = $conn->query($query);
$stats = $result->fetch();
echo $stats['count'] . ' updated records in period ' . $stats['start'] . ' to ' . $stats['end'] .'<br>';
-$query = 'SELECT count(*) as count FROM agency WHERE abn = \'0\'';
+/*$query = 'SELECT count(*) as count FROM agency WHERE abn = \'0\'';
$result = $conn->query($query);
$stats = $result->fetch();
-echo $stats['count'] . ' agencies with no ABN<br>';
+echo $stats['count'] . ' agencies with no ABN<br>';*/
-echo 'Last updated: '. $stats['importdate'].'<br>';
+echo 'Last updated: '. $stats['importday'].'<br>';
?>
--- a/displayCalendar.php
+++ b/displayCalendar.php
@@ -20,8 +20,8 @@
$query->execute();
databaseError($conn->errorInfo());
- MethodCountGraph($supplier);
- CnCGraph($supplier);
+ //MethodCountGraph($supplier);
+ //CnCGraph($supplier);
echo "<table> <thead>
<tr>
--- a/displaySupplier.php
+++ b/displaySupplier.php
@@ -5,9 +5,9 @@
include_header("Supplier");
$supplierS = htmlentities(strip_tags($_REQUEST['supplier']));
- MethodCountGraph($supplierS);
- CnCGraph($supplierS);
- MethodValueGraph($supplierS);
+ // MethodCountGraph($supplierS);
+ // CnCGraph($supplierS);
+ // MethodValueGraph($supplierS);
/* lobbyist ties
links to ABR/ASIC/Google News/ASX/Court records
@@ -27,12 +27,16 @@
echo $query;
$query = $conn->prepare($query);
-//$query->bindParam(":supplierName", $supplierName);
+
+ if ($supplierParts[0] > 0) {
$query->bindParam(":supplierABN", $supplierABN);
+ } else {
+ $query->bindParam(":supplierName", $supplierName);
+ }
$query->execute();
databaseError($conn->errorInfo());
- echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">';
- echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">';
+ // echo '<img src="graphs/displayMethodCountGraph.php?month=' . stripslashes($supplier) . '">';
+ // echo '<img src="graphs/displayCnCGraph.php?month=' . stripslashes($supplier) . '">';
echo "<table> <thead>
<tr>
--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -10,7 +10,9 @@
foreach ($unspscresult->fetchAll() as $row) {
$unspsc[$row['UNSPSC']] = $row['Title'];
}
-
+/*SELECT count(*), extract("week" from "publishDate") as week,extract("year" from "publishDate") as year
+FROM contractnotice
+where "childCN" is null group by week, year order by year, week*/
$query = $conn->prepare('
SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN",
EXTRACT(EPOCH FROM "publishDate") as "publishDate",
--- a/lib/graphs.inc.php
+++ b/lib/graphs.inc.php
@@ -142,7 +142,7 @@
global $conn;
includeFlot();
- $query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" WHERE "childCN" is null
+ $query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" WHERE "childCN" is null
GROUP BY abn ORDER BY SUM("value") DESC';
$query = $conn->prepare($query);
$query->execute();
@@ -201,11 +201,15 @@
;
function agencySuppliersGraph($agency) {
+
+ global $conn;
$agency = "AusAid";
$topX = 15;
- $query = 'SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND "childCN" is null AND agencyName = \'$agency\'
+ $query = 'SELECT SUM(value) as val, "supplierName" FROM contractnotice WHERE (extract ("YEAR" from "contractStart") >= :startYear) AND "childCN" is null AND "agencyName" = :agency
GROUP BY lower(supplierName) ORDER BY val DESC limit $topX';
$query = $conn->prepare($query);
+ $query->bindParam(":startYear",$startYear);
+ $query->bindParam(":agency",$agency);
$query->execute();
databaseError($conn->errorInfo());
$suppliers = Array();
@@ -216,9 +220,11 @@
}
- $query = 'SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND "childCN" is null and agencyName = \'$agency\'
-GROUP BY lower(supplierName) ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a';
- $query = $conn->prepare($query);
+ $query = 'SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, "supplierName" FROM contractnotice WHERE (extract ("YEAR" from "contractStart") >= :startYear) AND "childCN" is null and "agencyName" = :agency
+GROUP BY lower("supplierName") ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a';
+ $query = $conn->prepare($query);
+ $query->bindParam(":startYear",$startYear);
+ $query->bindParam(":agency",$agency);
$query->execute();
databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $row) {
@@ -230,8 +236,10 @@
}
function CnCGraph() {
- $query = "select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
-where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month";
+
+ global $conn;
+ $query = 'select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice
+where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month';
$query = $conn->prepare($query);
$query->execute();
databaseError($conn->errorInfo());
@@ -273,19 +281,21 @@
"Consultancies",
"Confidentialities"
);
- $query = 'SELECT \'consultancy\', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ consultancy=\'Yes\' AND "childCN" is null;';
+ $query = 'SELECT \'consultancy\', count(1) FROM contractnotice WHERE $agencyQ $supplierQ consultancy=\'Yes\' AND "childCN" is null;';
$result = $conn->query($query);
$row = $result->fetch(PDO::FETCH_ASSOC);
$attributes[0] = $row[1];
- $query = 'SELECT \'confidentiality\', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ (confidentialityContract=\'Yes\' OR confidentialityOutputs=\'Yes\') AND "childCN" is null;';
+ $query = 'SELECT \'confidentiality\', count(1) FROM contractnotice WHERE $agencyQ $supplierQ (confidentialityContract=\'Yes\' OR confidentialityOutputs=\'Yes\') AND "childCN" is null;';
$result = $conn->query($query);
$row = $result->fetch(PDO::FETCH_ASSOC);
$attributes[1] = $row[1];
}
function ContractPublishedGraph() {
+
+ global $conn;
$query = 'SELECT YEAR(publishDate), MONTH(publishDate),
-SUM(value) as val, count(1) as count FROM `contractnotice`
+SUM(value) as val, count(1) as count FROM contractnotice
WHERE (YEAR(publishDate) >= 2008)
AND "childCN" is null
GROUP BY MONTH(publishDate), YEAR(publishDate)
@@ -308,8 +318,10 @@
}
function ContractStartingGraph() {
+
+ global $conn;
$query = 'SELECT YEAR(contractStart), MONTH(contractStart),
-SUM(value) as val, count(1) as count FROM `contractnotice`
+SUM(value) as val, count(1) as count FROM contractnotice
WHERE (YEAR(contractStart) >= 2008)
AND "childCN" is null
GROUP BY MONTH(contractStart), YEAR(contractStart)
@@ -332,7 +344,9 @@
}
function MethodCountGraph() {
- $query = 'select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
+
+ global $conn;
+ $query = 'select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice
where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month';
$methods = Array("Direct", "Open", "Select");
$dates = Array();
@@ -367,7 +381,9 @@
}
function MethodValueGraph() {
- $query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice`
+
+ global $conn;
+ $query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice
where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month";
$methods = Array("Direct", "Open", "Select");
$dates = Array();