From: Alexander Sadleir Date: Wed, 21 Mar 2012 03:18:13 +0000 Subject: Fix CN frequency distribution graph X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=4f352a46bf37661beb195fdeff3b4d9d94225682 --- Fix CN frequency distribution graph --- --- a/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls +++ b/admin/data/14Nov11.ad604065e7caf17ec66e05a8b4932c73.xls @@ -145,7 +145,8 @@ ="Department of Foreign Affairs and Trade" ="" ="CN445119" 14-Nov-11 10:08 AM ="published" 01-Sep-11 31-Aug-12 300000.00 "Provision of News and Publicity Services" ="FO010911-CPDPMB0035" ="News and publicity services" ="Open" ="PMC2008 P0019" ="SON183611" ="No" ="" ="No" ="" ="No" ="" "" ="MEDIA MONITORS PTY LTD" ="131 Canberra Avenue" ="Griffith" ="2603" ="Australia" ="No" ="11002533851" ="FMB" ="CMD" ="0221" ="Defence Materiel Organisation" ="" ="CN445120" 14-Nov-11 10:08 AM ="published" 25-Oct-11 28-Nov-12 133650.00 " Provision of sheave for recovery assist system " ="DNL62K" ="Marine craft systems and subassemblies" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Beak Engineering" ="" ="Keysborough" ="3173" ="AUSTRALIA" ="No" ="35122373022" ="" ="" ="2600" ="Department of Foreign Affairs and Trade" ="" ="CN445121" 14-Nov-11 10:08 AM ="published" 05-Dec-11 30-Jun-12 28728.70 "Provision of Education and Training Services" ="CN051211-CMDSFB0324" ="Education and Training Services" ="Select" ="" ="No" ="" ="No" ="" ="No" ="" "" ="BERLITZ AUSTRALIA PTY" ="Ground Floor 35 Clarence Street" ="Sydney" ="2000" ="Australia" ="No" ="50007090271" ="FMB" ="CMD" ="0221" -="Department of Foreign Affairs and Trade" ="" ="CN445122" 14-Nov-11 10:08 AM ="published" 29-Aug-11 05-Sep-11 45669.80 "Provision of Management Advisory Services" ="CN290811-IMDIPB0102" ="Management advisory services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="THE BOSTON CONSULTING GROUP PTY LTD" ="Level 28 Chifley Tower 2 Chifley Square" ="Sydney" ="2000" ="Australia" ="No" ="70007347131" ="FMB" ="CMD" ="0221" +="Department of Foreign Affairs and Trade" ="" ="CN445122" 14-Nov-11 10:08 AM ="published" 29-Aug-11 05-Sep-11 45669.80 "Provision of Management Advisory Services" ="CN290811-IMDIPB0102" ="Management advisory services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="THE BOSTON CONSULTING GROUP PTY LTD" ="Level 28 Chifley Tower +2 Chifley Square" ="Sydney" ="2000" ="Australia" ="No" ="70007347131" ="FMB" ="CMD" ="0221" ="Department of Foreign Affairs and Trade" ="" ="CN445123" 14-Nov-11 10:08 AM ="published" 18-Oct-11 31-Dec-11 30000.00 "Provision of Security and Personal Safety" ="FO181011-CMDDSB0018" ="Security and personal safety" ="Open" ="DFAT09-DID-15" ="SON336277" ="No" ="" ="No" ="" ="No" ="" "" ="DYNAMIQ PTY LTD" ="Level 5, 33 York Street" ="Sydney" ="2000" ="Australia" ="No" ="38115069335" ="FMB" ="CMD" ="0221" ="Department of Foreign Affairs and Trade" ="" ="CN445124" 14-Nov-11 10:08 AM ="published" 06-Oct-11 30-Nov-11 44000.00 "Provision of Information Technology Consultation Services" ="CN061011-CMDFSB0028" ="Information technology consultation services" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for independent research or assessment" "" ="PHOENIX MANAGEMENT SERVICES PTY LTD" ="111 Settlers Road" ="Royalla" ="2620" ="Australia" ="No" ="11151887802" ="FMB" ="CMD" ="0221" ="Department of Innovation Industry Science and Research" ="CN316202" ="CN316202-A1" 30-Jul-10 04:52 PM 14-Nov-11 10:10 AM ="published" 01-Jul-10 30-Jun-11 229152.00 " Senior Sharepoint Developer " ="ORDER_DEED-001339-0" ="Temporary personnel services" ="Open" ="" ="SON26578" ="No" ="" ="No" ="" ="No" ="" "Extend engagement" ="MPM GROUP PTY LTD" ="Office 15, First floor Manuka Court" ="11-17 Bougainville Street, Manuka" ="2603" ="AUSTRALIA" ="No" ="62073795105" ="BUSINESS COLLABORATION BRANCH" ="CORPORATE" ="2600" @@ -264,7 +265,9 @@ ="National Archives of Australia" ="CN445191" ="CN445191-A2" 14-Nov-11 12:54 PM 14-Nov-11 03:17 PM ="published" 15-Sep-11 30-Dec-11 150127.00 " Provision of hardware and software services for multitouch interactive " ="POC00970-PO1100058" ="Software" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "incorrect value input on amendment No. 1" ="Lightwell Pty Ltd" ="Level 3, 48 Chippen St" ="CHIPPENDALE" ="2008" ="AUSTRALIA" ="No" ="84107866660" ="Corporate Services" ="National Archives" ="2600" ="Department of Foreign Affairs and Trade" ="" ="CN445250" 14-Nov-11 03:22 PM ="published" 01-Jul-11 30-Jun-12 798000.00 "Provision of Security and Control Equipment" ="CN010711-CMDDSB0327" ="Security and control equipment" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="SMITHS DETECTION PTY LTD" ="Unit 5, Botany Grove Estate, 14 Baker Street" ="Botany" ="2019" ="Australia" ="No" ="66088868062" ="FMB" ="CMD" ="0221" ="Department of Foreign Affairs and Trade" ="" ="CN445251" 14-Nov-11 03:22 PM ="published" 10-Nov-11 30-Jun-12 79500.00 "Provision of Software" ="CN101111-IMDISB0238" ="Software" ="Direct" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="I.T. EXECUTIVE CONSULTING PTY. LIMITED" ="Suite 283, 4 Young Street" ="Neutral Bay" ="2089" ="Australia" ="No" ="24096436001" ="FMB" ="CMD" ="0221" -="Department of Foreign Affairs and Trade" ="" ="CN445252" 14-Nov-11 03:22 PM ="published" 09-Nov-11 31-Mar-12 55000.00 "Provision of Fleet Management Services" ="CN091111-IMDISB0017" ="Fleet management services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="INFINITE CONSULTING PTY LIMITED" ="Phoenix House Ground Floor Unit 8 86-88 Northbourne Ave" ="Braddon" ="2612" ="Australia" ="No" ="56081954645" ="FMB" ="CMD" ="0221" +="Department of Foreign Affairs and Trade" ="" ="CN445252" 14-Nov-11 03:22 PM ="published" 09-Nov-11 31-Mar-12 55000.00 "Provision of Fleet Management Services" ="CN091111-IMDISB0017" ="Fleet management services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="INFINITE CONSULTING PTY LIMITED" ="Phoenix House +Ground Floor Unit 8 +86-88 Northbourne Ave" ="Braddon" ="2612" ="Australia" ="No" ="56081954645" ="FMB" ="CMD" ="0221" ="Australian Federal Police" ="CN412688" ="CN412688-A1" 22-Jul-11 09:05 AM 14-Nov-11 03:25 PM ="published" 01-Jul-11 14-Aug-11 32611.87 " Provision of high level technical support and management services of the UNIX environment " ="1015592" ="Computer services" ="Open" ="RFT 8-2007" ="SON85958" ="No" ="" ="No" ="" ="No" ="" "Contract terminated and value decreased by $220,546.53" ="Paxus Australia Pty Limited" ="" ="Braddon" ="2601" ="AUSTRALIA" ="No" ="35004609616" ="National Procurement and Contracts" ="" ="2600" ="Australian Federal Police" ="CN113130" ="CN113130-A6" 21-Aug-08 08:18 AM 14-Nov-11 03:33 PM ="published" 01-Jul-08 30-Jun-11 785335.94 " Provision of a senior database administrator " ="1011258" ="Computer services" ="Open" ="RFT 8-2007" ="SON85958" ="No" ="" ="No" ="" ="No" ="" "Agreement have ceased and value reduced by $18,913.66" ="Paxus Australia Pty Limited" ="" ="Dickson" ="2612" ="AUSTRALIA" ="No" ="35004609616" ="National Procurement and Contracts" ="" ="2600" ="Department of Broadband Communications and the Digital Economy" ="" ="CN445255" 14-Nov-11 03:33 PM ="published" 11-Nov-11 31-Jan-12 16500.00 "Legal Services - Constitutional Advice" ="0004601966" ="Legal services" ="Open" ="DCON/10/65" ="SON347233" ="Yes" ="Intellectual property" ="Yes" ="Intellectual property" ="Yes" ="Need for specialised or professional skills" "" ="AUSTRALIAN GOVERNMENT SOLICITOR" ="Locked Bag 7246" ="Canberra Mail Centre" ="2610" ="Australia" ="No" ="69405937639" ="" ="LEGAL Legal Group" ="2603" --- a/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls +++ b/admin/data/21Nov11.60bea7514275085b1cf63f231ac9094a.xls @@ -85,7 +85,7 @@ ="Department of the Prime Minister and Cabinet" ="" ="CN447929" 24-Nov-11 03:33 PM ="published" 01-Aug-11 30-Apr-13 588472.00 " Travel Management Services " ="PMC 2011 TMP" ="Travel facilitation" ="Open" ="FIN09/AMG007" ="SON289574" ="No" ="" ="No" ="" ="No" ="" "" ="Australian OpCo Pty Ltd T/A FCm Travel Solutions" ="Level 2, 545 Queen St" ="Brisbane" ="4000" ="AUSTRALIA" ="No" ="20003279534" ="FMB" ="CSD" ="2600" ="AusAid" ="CN330933" ="CN330933-A1" 14-Sep-10 11:16 AM 22-Nov-11 01:33 PM ="published" 17-May-10 31-Oct-11 235400.00 "MFD Printers - Lease & DocuCare" ="D00209" ="Computer printers" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "value & duration" ="FUJI XEROX" ="" ="NORTH RYDE" ="2113" ="AUSTRALIA" ="No" ="63000341819" ="" ="" ="2601" ="Australian Transaction Reports and Analysis Centre (AUSTRAC)" ="" ="CN443393" 21-Nov-11 11:13 AM ="published" 13-Jul-11 13-Jul-11 14085.50 " Provision of training " ="Sept 2011" ="Education and Training Services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Object Consulting" ="" ="North Sydney" ="2060" ="AUSTRALIA" ="No" ="55003682693" ="" ="" ="2067" -="AusAid" ="" ="CN443720" 22-Nov-11 01:32 PM ="published" 07-Oct-11 30-Nov-11 53493.00 " Review of the Australian aid program's food security and rural development portfolio " ="D48944/4" ="Environmental management" ="Open" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="Griffin NRM Pty. Ltd." ="" ="Garran" ="2605" ="AUSTRALIA" ="No" ="67088471523" ="" ="" ="2601" +="AusAid" ="" ="CN443720" 22-Nov-11 01:32 PM ="published" 07-Oct-11 30-Nov-11 53493.00 " Review of the Australian aid program's food security and rural development portfolio " ="D48944/4" ="Environmental management" ="Open" ="" ="No" ="" ="No" ="" ="Yes" ="Need for specialised or professional skills" "" ="Griffin NRM Pty. Ltd. Griffin NRM Pty Ltd" ="" ="Garran" ="2605" ="AUSTRALIA" ="No" ="67088471523" ="" ="" ="2601" ="Department of Families, Housing, Community Services and Indigenous Affairs" ="CN395069" ="CN395069-A1" 09-Jun-11 02:12 PM 21-Nov-11 09:53 AM ="published" 08-Jun-11 31-Dec-11 26598.00 " Creative and Design Services " ="45395003" ="Graphic design" ="Open" ="07/T585" ="No" ="" ="No" ="" ="No" ="" "Decrease of Contract Value" ="David & Kate Pty Ltd t/aDogma" ="" ="East St Kilda" ="3183" ="AUSTRALIA" ="No" ="26613552343" ="Communications and Media" ="FaHCSIA" ="2900" ="Department of Families, Housing, Community Services and Indigenous Affairs" ="" ="CN444996" 21-Nov-11 02:58 PM ="published" 01-Dec-11 15-Mar-12 10194.80 " Print Closing the Gap Prime Minister's Report 2012 " ="45405148" ="Printing" ="Open" ="08/T629" ="SON206666" ="No" ="" ="No" ="" ="No" ="" "" ="NEW MILLENNIUM PRINT PTY LTD" ="" ="FYSHWICK" ="2609" ="AUSTRALIA" ="No" ="69089018271" ="" ="" ="2900" ="Department of Families, Housing, Community Services and Indigenous Affairs" ="CN402123" ="CN402123-A1" 14-Jul-11 02:24 PM 24-Nov-11 03:58 PM ="published" 01-Jul-11 31-Mar-12 79568.00 " Human Resource Processes " ="45396974" ="Audit services" ="Open" ="07/T609" ="SON90044" ="No" ="" ="No" ="" ="No" ="" "variation to extend timeframe" ="Oakton Services Pty Ltd" ="" ="Kingston" ="2604" ="AUSTRALIA" ="No" ="31100103268" ="Legal & Compliance" ="FaHCSIA" ="2900" --- a/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls +++ b/admin/data/27Sep11.986583fa02be5a70c20d4f595d4f8d1b.xls @@ -587,7 +587,8 @@ ="Department of Health and Ageing - Therapeutic Goods Administration" ="CN428386" ="CN428386-A1" 09-Sep-11 11:35 AM 29-Feb-12 02:29 PM ="published" 12-Sep-11 20-Dec-11 39562.08 " Engagement of a Contractor - Events & Communications Officer " ="TGA2011-0343" ="Temporary personnel services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "Extension of contract" ="Careers Unlimited Pty Ltd" ="35-37 London Circuit" ="Canberra City" ="2601" ="AUSTRALIA" ="No" ="17075304737" ="Office of Parliamentary & Strategic Support" ="Therapeutic Goods Administration" ="2606" ="Department of Defence" ="" ="CN467784" 29-Feb-12 02:36 PM ="published" 28-Feb-12 28-Mar-12 12074.28 " Spare Parts Military Vehicles " ="EB5545" ="Motor vehicles" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="JAGUAR LAND ROVER AUSTRALIA" ="" ="TULLAMARINE" ="3043" ="AUSTRALIA" ="No" ="86004352238" ="" ="" ="2600" ="Department of Foreign Affairs and Trade" ="" ="CN467786" 29-Feb-12 02:39 PM ="published" 22-Feb-12 21-Feb-15 149669.30 "Provision of Communications Devices and Accessories" ="CN220212-IMDIPB0047" ="Communications Devices and Accessories" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="VOCALITY PTY LTD" ="4/19 Reliance Drive" ="Tuggerah" ="2259" ="Australia" ="No" ="61103706369" ="FMB" ="CMD" ="0221" -="Department of Foreign Affairs and Trade" ="" ="CN467787" 29-Feb-12 02:39 PM ="published" 24-Feb-12 06-Mar-12 56000.00 "Provision of Public Relation Services" ="CN240212-SMDMEB0016" ="Public relation services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="AUSTRALIA GULF COUNCIL LIMITED" ="Level 7 8 Spring Street" ="Sydney" ="2000" ="Australia" ="No" ="99143749786" ="FMB" ="CMD" ="0221" +="Department of Foreign Affairs and Trade" ="" ="CN467787" 29-Feb-12 02:39 PM ="published" 24-Feb-12 06-Mar-12 56000.00 "Provision of Public Relation Services" ="CN240212-SMDMEB0016" ="Public relation services" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "" ="AUSTRALIA GULF COUNCIL LIMITED" ="Level 7 +8 Spring Street" ="Sydney" ="2000" ="Australia" ="No" ="99143749786" ="FMB" ="CMD" ="0221" ="Department of Health and Ageing - Therapeutic Goods Administration" ="" ="CN467788" 29-Feb-12 02:44 PM ="published" 17-Feb-10 16-Feb-13 990000.00 " Guarding and Patrol Services at TGA " ="TGA2012-0094" ="Security guard services" ="Open" ="" ="No" ="" ="No" ="" ="No" ="" "" ="MSS Security Australia Pty Ltd" ="149-155 Milton Street" ="Ashfield" ="2131" ="AUSTRALIA" ="No" ="29100573966" ="Office of Corporate Services" ="Therapeutic Goods Administration" ="2606" ="Defence Materiel Organisation" ="" ="CN467789" 29-Feb-12 02:47 PM ="published" 28-Sep-11 03-Oct-11 50168.70 " Lubricating oil " ="JW1EPX/3073" ="Lubricants and oils and greases and anti corrosives" ="Open" ="" ="No" ="" ="No" ="" ="No" ="" "" ="BP Australia Ltd" ="" ="Docklands" ="3008" ="AUSTRALIA" ="No" ="53004085616" ="JFLA" ="MSD" ="2000" ="Defence Materiel Organisation" ="CN465897" ="CN465897-A1" 21-Feb-12 08:23 AM 29-Feb-12 02:49 PM ="published" 20-Feb-12 14-May-12 15534.20 " Lift Motor Vehicle 5 Tonne Capacity " ="KM0AER" ="Tools and General Machinery" ="Direct" ="" ="No" ="" ="No" ="" ="No" ="" "Change of Agency Reference ID" ="Garage Equipment" ="17 Canberra Street" ="Hemmant" ="4174" ="AUSTRALIA" ="No" ="75820442568" ="LSSB" ="LSD" ="3006" --- a/admin/import.php +++ /dev/null @@ -1,263 +1,1 @@ - $f) { - $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; -} -$contractNoticeInsertQ.= ");"; -$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); - -$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( '; -foreach ($supplierFields as $key => $f) { - $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?"; -} -$supplierInsertQ.= ");"; -$supplierInsertQ = $conn->prepare($supplierInsertQ); - -$agencyInsertQ = 'INSERT INTO agency_nametoabn ("' . implode('" , "', $agencyFields) . '") VALUES ( '; -foreach ($agencyFields as $key => $f) { - $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?"; -} -$agencyInsertQ.= ");"; -$agencyInsertQ = $conn->prepare($agencyInsertQ); - -function processFile($fpath, $tablename) { - global $conn, $contractNoticeFields, $contractNoticeInsertQ, $supplierFields, $supplierInsertQ,$agencyFields, $agencyInsertQ; - $row = 1; - $handle = fopen($fpath, "r"); - //"t" mode string translates windows line breaks to unix - $datamapping0711 = array( - "Agency" => "agencyName", - "Parent CN ID" => "parentCN", - "CN ID" => "CNID", - "Publish Date" => "publishDate", - "Amendment Date" => "amendDate", - "Status" => "", - "StartDate" => "contractStart", - "EndDate" => "contractEnd", - "Value" => "value", - "Description" => "description", - "Agency Ref Id" => "agencyID", - "Category" => "category", - "Procurement Method" => "procurementMethod", - "ATM ID" => "atmID", - "SON ID" => "SONID", - "Confidentiality - Contract" => "confidentialityContract", - "Confidentiality - Contract Reason(s)" => "confidentialityContractReason", - "Confidentiality - Outputs" => "confidentialityOutputs", - "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason", - "Consultancy" => "consultancy", - "Consultancy Reason(s)" => "consultancyReason", - "Amendment Reason" => "amendmentReason", - "Supplier Name" => "supplierName", - "Supplier Address" => "supplierAddress", - "Supplier City" => "supplierCity", - "Supplier Postcode" => "supplierPostcode", - "Supplier Country" => "supplierCountry", - "Supplier ABNExempt" => "supplierABNExempt", - "Supplier ABN" => "supplierABN", - "Agency Branch" => "contactBranch", - "Agency Divison" => "contactDivision", - "Agency Postcode" => "contactPostcode", - "" => "" - ); - $headers; - - while (($data = fgetcsv($handle, 1000, "\t")) !== false) { - $num = count($data); - if ($row == 3) { - $headers = $data; - } elseif ($row > 3) { - if ($num > count($datamapping0711)) { - die("Error in data import; data mapping fields out of bounds or changed
" . $fname . print_r($data)); - } - $contractNoticeInsert = Array(); - $supplierInsert = Array(); - $agencyInsert = Array(); - $contractNoticeInsert[] = $fpath; - $keys = array_keys($datamapping0711); - for ($c = 0; $c < $num; $c++) { - $data[$c] = trim($data[$c], "="); - $data[$c] = trim($data[$c], "\""); - if ($tablename == "contractnotice") { - if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) { - if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") { - $data[$c] = substr($data[$c], 2); // take off the "CN" prefix - if ($data[$c] > 0 && $data[$c] != '0') { - $contractNoticeInsert[] = $data[$c]; - } else { - $contractNoticeInsert[] = null; - } - } elseif (($datamapping0711[$headers[$c]]) == "supplierABN") { - if ($data[$c] > 0 && $data[$c] != '0') { - $contractNoticeInsert[] = $data[$c]; - } else { - $contractNoticeInsert[] = null; - } - } elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") { - $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c])); - } else { - if (strstr("\" =", $data[$c] > 0)) { - die("Invalid Description field" . $contractNoticeInsert); - } - $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c])); - - $contractNoticeInsert[] = $colvalue; - } - } - } else if ($tablename == "supplierdetails") { - if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) { - if (($datamapping0711[$headers[$c]]) == "supplierABN") { - if ($data[$c] > 0) { - $contractNoticeInsert[] = $data[$c]; - } else { - $contractNoticeInsert[] = 0; - } - } else { - $supplierInsert[] = $data[$c]; - } - } - } else if ($tablename == "agency") { - if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) { - $agencyInsert[] = $data[$c]; - } - } - } - flush(); - if ($tablename == "contractnotice") { - $contractNoticeInsertQ->execute($contractNoticeInsert); - $errors = $conn->errorInfo(); - if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { - - } elseif ($errors[1] == 0) { - $success++; - } else { - foreach ($contractNoticeFields as $key => $cnf) { - echo var_dump($contractNoticeInsert[$key]) . $cnf . "
"; - } - echo $data[2] . " failed CN insert.
" . print_r($errors, true) . "
row $row

\n"; - } - } else if ($tablename == "supplierdetails") { - $supplierInsertQ->execute($supplierInsert); - $errors = $conn->errorInfo(); - if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { - - } elseif ($errors[1] == 0) { - $success++; - } else { - echo $data[2] . " failed supplier insert.
" . print_r($errors, true) . "
" . print_r($supplierInsert, true) . "
$row

\n"; - } - } else if ($tablename == "agency") { - $agencyInsertQ->execute($agencyInsert); - $errors = $conn->errorInfo(); - if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { - - } elseif ($errors[1] == 0) { - $success++; - } else { - echo $data[2] . " failed agency insert.
" . print_r($errors, true) . "
" . print_r($agencyInsert, true) . "
$row

\n"; - } - } - flush(); - //echo "
\n"; - } - $row++; - } - fclose($handle); - $contractNoticeInsertQ->closeCursor(); - $supplierInsertQ->closeCursor(); - $agencyInsertQ->closeCursor(); - - return $success; -} - -$path = 'data/'; -if ($_REQUEST["fname"] == "") { - echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list
"; - $dhandle = opendir($path); - // define an array to hold the files - $files = array(); - if ($dhandle) { - // loop through all of the files - while (false !== ($fname = readdir($dhandle))) { - if (($fname != '.') && ($fname != '..')) { - $files[date("c", filemtime($path . $fname)).md5($fname)] = $fname; - } - } - } - ksort($files); - foreach ($files as $date => $fname) { - echo "$fname " . filesize($path . $fname) . " " . $date . "
"; - - } -} else { - $success = 0; - $fname = $_REQUEST["fname"]; - echo " ============== $fname ==============
"; - flush(); - $success+= processFile($path . $fname, "contractnotice"); - $success+= processFile($path . $fname, "agency"); - $success+= processFile($path . $fname, "supplier"); - echo "
$success records successfully created"; - - flush(); - // run post import data processing -// cn - echo "link amend
"; - include ("linkAmendments.php"); - echo "update UNSPSC
"; - include ("updateUNSPSC.php"); -// agency -//include ("setAgencyStatus.php"); -//include ("setAgencyURLABN.php"); -} -?> - --- a/admin/linkAmendments.php +++ b/admin/linkAmendments.php @@ -1,8 +1,5 @@ prepare($query); - $query->execute(); $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" WHERE c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; @@ -10,7 +7,6 @@ $query->execute(); databaseError($conn->errorInfo()); foreach ($query->fetchAll() as $row) { - if ($row['parentCN'] != 0) { $conn->exec('UPDATE contractnotice SET "childCN" = \'' . $row['CNID'] . '\' where "CNID" = \'' . $row['parentCN'] . '\';'); @@ -23,7 +19,7 @@ echo $row['CNID'] . " linked to parent " . $row['parentCN'] . "
\n"; else print_r($errors); - } + } // also need to eliminate CN 100528/100529 - check for double parent CNs with no childCN, latest sequent CN id keeps childCN = 0 $query = 'SELECT "parentCN", array_agg("CNID"), count(*) from contractnotice WHERE "parentCN" IN --- a/admin/neo4jimporter/pom.xml +++ b/admin/neo4jimporter/pom.xml @@ -7,7 +7,7 @@ org.neo4j neo4j - 1.6.1 + 1.5 postgresql --- a/admin/neo4jimporter/src/main/java/Importer.java +++ b/admin/neo4jimporter/src/main/java/Importer.java @@ -88,26 +88,26 @@ + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", " + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); String previousAgency = ""; - GraphDatabaseService gds = inserter.getGraphDbService(); - HashMap supplierIDs = new HashMap(); - HashMap agencyIDs = new HashMap(); - +GraphDatabaseService gds = inserter.getGraphDbService(); +HashMap supplierIDs = new HashMap(); +HashMap agencyIDs = new HashMap(); + // Loop through the result set while (rs.next()) { long supplierID, agencyID; String supplierKey; - if (agencyIDs.get(rs.getString("agencyName")) == null) { - Node myNode = gds.createNode(); - myNode.setProperty("Label", rs.getString("agencyName")); - myNode.setProperty("type", "agency"); - agencyIDs.put(rs.getString("agencyName"), myNode.getId()); - if (myNode.getId() % 100 == 0) { - System.out.println("Agency " + myNode.getId()); - } - } - agencyID = agencyIDs.get(rs.getString("agencyName")); - - + if (agencyIDs.get(rs.getString("agencyName")) == null) { + Node myNode = gds.createNode(); + myNode.setProperty("Label", rs.getString("agencyName")); + myNode.setProperty("type", "agency"); + agencyIDs.put(rs.getString("agencyName"), myNode.getId()); + if (myNode.getId() %100 == 0) { + System.out.println("Agency "+myNode.getId()); +} + } + agencyID = agencyIDs.get(rs.getString("agencyName")); + + if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { supplierKey = rs.getString("supplierABN"); } else { @@ -115,19 +115,19 @@ } // inject some data if (supplierIDs.get(supplierKey) == null) { - Node myNode = gds.createNode(); - myNode.setProperty("Label", rs.getString("supplierName")); - myNode.setProperty("type", "supplier"); + Node myNode = gds.createNode(); + myNode.setProperty("Label", rs.getString("supplierName")); + myNode.setProperty("type", "supplier"); supplierIDs.put(supplierKey, myNode.getId()); - if (myNode.getId() % 1000 == 0) { - System.out.println("Supplier " + myNode.getId()); - } + if (myNode.getId() %1000 == 0) { + System.out.println("Supplier "+myNode.getId()); +} } supplierID = supplierIDs.get(supplierKey); long rel = inserter.createRelationship(agencyID, supplierID, - DynamicRelationshipType.withName("KNOWS"), null); + DynamicRelationshipType.withName("KNOWS"), null); inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); } @@ -154,4 +154,6 @@ indexProvider.shutdown(); inserter.shutdown(); } + + } --- /dev/null +++ b/admin/partialdata/scraper.txt @@ -1,1 +1,74 @@ += 1024 && $i < 4; $i++) $size /= 1024; + return round($size, 2).$units[$i]; +} +$days = 4; +if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; +$startDate = strtotime("05-Jun-2008"); +if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; + +function getFile($startDate, $days, $minVal, $maxVal) { +global $split; + $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); +$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; +echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; +$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; +echo ""; +$current = file_get_contents($url); +if (strpos($current,"There are no results that match your selection.")> 0 ) { + echo "Empty file!
"; +} +if (strpos($current,"Your search returned more than 1000 results.") === false) { + file_put_contents($file, $current); + echo "$file saved
"; + echo format_bytes(filesize($file))."
"; + echo 'Load next '.($days).' days
'; + echo 'Load next '.($days*2).' days
'; + echo 'Load next '.($days).' days with split
'; + flush(); +if (!isset($_REQUEST['split']) && !$split) { +echo "Success so fetching next $days...
"; +getFile($endDate, $days, "" , ""); +} + return true; +} else { + echo "Too many records!
"; + echo 'Load '.($days/2).' days instead?
'; + echo 'Split instead?
'; + flush(); +if (!isset($_REQUEST['split']) && !$split) { +echo "Failure so splitting ...
"; + doSplit($startDate, $days); +} + return false; +} +} +function doSplit($startDate, $days) { +global $split; +$split = true; +set_time_limit(20); +getFile($startDate, $days, 0, 12000); +getFile($startDate, $days, 12000, 16000); + getFile($startDate, $days, 16000, 20000); + getFile($startDate, $days, 20000, 30000); + getFile($startDate, $days, 30000, 40000); +// getFile($startDate, $days, 40000, 80000); + getFile($startDate, $days, 40000, 60000); + getFile($startDate, $days, 60000, 80000); +// getFile($startDate, $days, 80000, 300000); + getFile($startDate, $days, 80000, 150000); + getFile($startDate, $days, 150000, 300000); + getFile($startDate, $days, 300000, 999999999); +} +if (isset($_REQUEST['split'])) { + doSplit($startDate, $days); +} else { + getFile($startDate, $days, "" , ""); +} +?> + --- a/admin/updateReport.php +++ b/admin/updateReport.php @@ -1,24 +1,24 @@ query($query); $stats = $result->fetch(); echo $stats['count'] . ' new records in period ' . $stats['start'] . ' to ' . $stats['end'] .'
'; -$query = 'SELECT max(DATE("importDate")) as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end +$query = 'SELECT DATE("importDate") as importday, count(*) as count, min("publishDate") as start, max("publishDate") as end FROM contractnotice WHERE ("childCN" is not null OR "parentCN" is not null) AND DATE("importDate") in (select * from (SELECT DATE("importDate") -FROM contractnotice ORDER BY "importDate" DESC limit 1) alias)'; +FROM contractnotice ORDER BY importDate DESC limit 1) alias)'; $result = $conn->query($query); $stats = $result->fetch(); echo $stats['count'] . ' updated records in period ' . $stats['start'] . ' to ' . $stats['end'] .'
'; -/*$query = 'SELECT count(*) as count FROM agency WHERE abn = \'0\''; +$query = 'SELECT count(*) as count FROM agency WHERE abn = \'0\''; $result = $conn->query($query); $stats = $result->fetch(); -echo $stats['count'] . ' agencies with no ABN
';*/ +echo $stats['count'] . ' agencies with no ABN
'; -echo 'Last updated: '. $stats['importday'].'
'; +echo 'Last updated: '. $stats['importdate'].'
'; ?> --- a/displayCalendar.php +++ b/displayCalendar.php @@ -20,8 +20,8 @@ $query->execute(); databaseError($conn->errorInfo()); - //MethodCountGraph($supplier); - //CnCGraph($supplier); + MethodCountGraph($supplier); + CnCGraph($supplier); echo " --- a/exportData.csv.php +++ b/exportData.csv.php @@ -10,9 +10,7 @@ foreach ($unspscresult->fetchAll() as $row) { $unspsc[$row['UNSPSC']] = $row['Title']; } -/*SELECT count(*), extract("week" from "publishDate") as week,extract("year" from "publishDate") as year -FROM contractnotice -where "childCN" is null group by week, year order by year, week*/ + $query = $conn->prepare(' SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN", EXTRACT(EPOCH FROM "publishDate") as "publishDate", --- a/lib/graphs.inc.php +++ b/lib/graphs.inc.php @@ -27,7 +27,11 @@ var d1 = []; var d2 = []; prepare($query); $query->execute(); $errors = $conn->errorInfo(); @@ -40,7 +44,9 @@ echo "d1.push([ " . intval($delta['cnid']) . ", " . intval($delta['count']) . "]); \n"; }; - $query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid'; + $query = 'select cnid, count(*) from (select ("CNID"::integer - MOD("CNID"::integer,100)) as cnid + from contractnotice where "CNID" not like \'%-A%\' and "parentCN" is not null) + as a group by cnid order by cnid'; $query = $conn->prepare($query); $query->execute(); $errors = $conn->errorInfo(); @@ -71,7 +77,7 @@ points: { show: true } }, bars: { show: true } - }, + }] ]; var options = { @@ -136,7 +142,7 @@ global $conn; includeFlot(); - $query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" WHERE "childCN" is null + $query = 'SELECT SUM("value") as val, MAX(contractnotice."agencyName") as agencyname FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" WHERE "childCN" is null GROUP BY abn ORDER BY SUM("value") DESC'; $query = $conn->prepare($query); $query->execute(); @@ -195,15 +201,11 @@ ; function agencySuppliersGraph($agency) { - - global $conn; $agency = "AusAid"; $topX = 15; - $query = 'SELECT SUM(value) as val, "supplierName" FROM contractnotice WHERE (extract ("YEAR" from "contractStart") >= :startYear) AND "childCN" is null AND "agencyName" = :agency + $query = 'SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND "childCN" is null AND agencyName = \'$agency\' GROUP BY lower(supplierName) ORDER BY val DESC limit $topX'; $query = $conn->prepare($query); - $query->bindParam(":startYear",$startYear); - $query->bindParam(":agency",$agency); $query->execute(); databaseError($conn->errorInfo()); $suppliers = Array(); @@ -214,11 +216,9 @@ } - $query = 'SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, "supplierName" FROM contractnotice WHERE (extract ("YEAR" from "contractStart") >= :startYear) AND "childCN" is null and "agencyName" = :agency -GROUP BY lower("supplierName") ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a'; - $query = $conn->prepare($query); - $query->bindParam(":startYear",$startYear); - $query->bindParam(":agency",$agency); + $query = 'SELECT sum(a.val) as value, count(1) as count from (SELECT SUM(value) as val, supplierName FROM `contractnotice` WHERE (YEAR(contractStart) >= $startYear) AND "childCN" is null and agencyName = \'$agency\' +GROUP BY lower(supplierName) ORDER BY val DESC LIMIT 18446744073709551610 OFFSET $topX) as a'; + $query = $conn->prepare($query); $query->execute(); databaseError($conn->errorInfo()); foreach ($query->fetchAll() as $row) { @@ -230,10 +230,8 @@ } function CnCGraph() { - - global $conn; - $query = 'select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice -where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month'; + $query = "select procurementMethod, count(1) as count, SUM(value) as val, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` +where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month"; $query = $conn->prepare($query); $query->execute(); databaseError($conn->errorInfo()); @@ -275,21 +273,19 @@ "Consultancies", "Confidentialities" ); - $query = 'SELECT \'consultancy\', count(1) FROM contractnotice WHERE $agencyQ $supplierQ consultancy=\'Yes\' AND "childCN" is null;'; + $query = 'SELECT \'consultancy\', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ consultancy=\'Yes\' AND "childCN" is null;'; $result = $conn->query($query); $row = $result->fetch(PDO::FETCH_ASSOC); $attributes[0] = $row[1]; - $query = 'SELECT \'confidentiality\', count(1) FROM contractnotice WHERE $agencyQ $supplierQ (confidentialityContract=\'Yes\' OR confidentialityOutputs=\'Yes\') AND "childCN" is null;'; + $query = 'SELECT \'confidentiality\', count(1) FROM `contractnotice` WHERE $agencyQ $supplierQ (confidentialityContract=\'Yes\' OR confidentialityOutputs=\'Yes\') AND "childCN" is null;'; $result = $conn->query($query); $row = $result->fetch(PDO::FETCH_ASSOC); $attributes[1] = $row[1]; } function ContractPublishedGraph() { - - global $conn; $query = 'SELECT YEAR(publishDate), MONTH(publishDate), -SUM(value) as val, count(1) as count FROM contractnotice +SUM(value) as val, count(1) as count FROM `contractnotice` WHERE (YEAR(publishDate) >= 2008) AND "childCN" is null GROUP BY MONTH(publishDate), YEAR(publishDate) @@ -312,10 +308,8 @@ } function ContractStartingGraph() { - - global $conn; $query = 'SELECT YEAR(contractStart), MONTH(contractStart), -SUM(value) as val, count(1) as count FROM contractnotice +SUM(value) as val, count(1) as count FROM `contractnotice` WHERE (YEAR(contractStart) >= 2008) AND "childCN" is null GROUP BY MONTH(contractStart), YEAR(contractStart) @@ -338,9 +332,7 @@ } function MethodCountGraph() { - - global $conn; - $query = 'select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice + $query = 'select procurementMethod, count(1) as count, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month'; $methods = Array("Direct", "Open", "Select"); $dates = Array(); @@ -375,9 +367,7 @@ } function MethodValueGraph() { - - global $conn; - $query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from contractnotice + $query = "select procurementMethod, SUM(value) as value, MONTH(contractStart) as month, YEAR(contractStart) as year from `contractnotice` where $agencyQ $supplierQ $standardQ group by procurementMethod,year,month order by procurementMethod,year,month"; $methods = Array("Direct", "Open", "Select"); $dates = Array();