From: Maxious Date: Fri, 23 Dec 2011 00:23:24 +0000 Subject: Migrate agencies to 2 tables X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=c3370a6d6f329795a5c095574962d8faa08b1504 --- Migrate agencies to 2 tables --- --- a/admin/fixoldamend.php +++ b/admin/fixoldamend.php @@ -20,7 +20,7 @@ echo "$oldCN => $newCN (from parent CN $parentCN)
\n"; } else { - echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']}
\n"; +// echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']}
\n"; } } --- a/admin/linkAmendments.php +++ b/admin/linkAmendments.php @@ -2,7 +2,7 @@ include_once ("../lib/common.inc.php"); $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" WHERE -c."parentCN" > 0 AND p."childCN" IS NULL '; +c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; $query = $conn->prepare($query); $query->execute(); databaseError($conn->errorInfo()); @@ -28,9 +28,9 @@ FROM contractnotice GROUP BY "parentCN" HAVING COUNT(*) > 1 - AND "parentCN" != 0 + AND "parentCN" IS NOT NULL ) -AND "childCN" = 0 +AND "childCN" IS NULL GROUP BY "parentCN" having count(*) > 1'; $query = $conn->prepare($query); $query->execute(); --- a/admin/partialdata/scraper.php +++ /dev/null @@ -1,74 +1,1 @@ -= 1024 && $i < 4; $i++) $size /= 1024; - return round($size, 2).$units[$i]; -} -$days = 4; -if (isset($_REQUEST['days'])) $days = $_REQUEST['days']; -$startDate = strtotime("05-Jun-2008"); -if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate']; - -function getFile($startDate, $days, $minVal, $maxVal) { -global $split; - $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days"); -$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls'; -echo "Fetching $file ($days days) ($minVal < value < $maxVal )... "; -$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results"; -echo ""; -$current = file_get_contents($url); -if (strpos($current,"There are no results that match your selection.")> 0 ) { - echo "Empty file!
"; -} -if (strpos($current,"Your search returned more than 1000 results.") === false) { - file_put_contents($file, $current); - echo "$file saved
"; - echo format_bytes(filesize($file))."
"; - echo 'Load next '.($days).' days
'; - echo 'Load next '.($days*2).' days
'; - echo 'Load next '.($days).' days with split
'; - flush(); -if (!isset($_REQUEST['split']) && !$split) { -echo "Success so fetching next $days...
"; -getFile($endDate, $days, "" , ""); -} - return true; -} else { - echo "Too many records!
"; - echo 'Load '.($days/2).' days instead?
'; - echo 'Split instead?
'; - flush(); -if (!isset($_REQUEST['split']) && !$split) { -echo "Failure so splitting ...
"; - doSplit($startDate, $days); -} - return false; -} -} -function doSplit($startDate, $days) { -global $split; -$split = true; -set_time_limit(20); -getFile($startDate, $days, 0, 12000); -getFile($startDate, $days, 12000, 16000); - getFile($startDate, $days, 16000, 20000); - getFile($startDate, $days, 20000, 30000); - getFile($startDate, $days, 30000, 40000); -// getFile($startDate, $days, 40000, 80000); - getFile($startDate, $days, 40000, 60000); - getFile($startDate, $days, 60000, 80000); -// getFile($startDate, $days, 80000, 300000); - getFile($startDate, $days, 80000, 150000); - getFile($startDate, $days, 150000, 300000); - getFile($startDate, $days, 300000, 999999999); -} -if (isset($_REQUEST['split'])) { - doSplit($startDate, $days); -} else { - getFile($startDate, $days, "" , ""); -} -?> - --- a/admin/updateUNSPSC.php +++ b/admin/updateUNSPSC.php @@ -19,13 +19,26 @@ $unspsc[$armor] = $row['UNSPSC']; $erre = str_replace("er", "re", $row['Title']); $unspsc[$erre] = $row['UNSPSC']; + $center = str_replace("center", "centre", $row['Title']); + $unspsc[$center] = $row['UNSPSC']; + $accessory = str_replace("accesor", "accessor", $row['Title']); + $unspsc[$accessory] = $row['UNSPSC']; + $lyslyz = str_replace("lyz", "lys", $row['Title']); + $unspsc[$lyslyz] = $row['UNSPSC']; $tire = str_replace("ire", "yre", $row['Title']); $unspsc[$tire] = $row['UNSPSC']; - $aeroplane = str_replace("airplane","aeroplane", $row['Title']); - $unspsc[$aeroplane] = $row['UNSPSC']; - $lyslyz = str_replace("lyz", "lys", $row['Title']); - $unspsc[$lyslyz] = $row['UNSPSC']; + + $pe = str_replace("pe", "pae", $row['Title']); + $unspsc[$pe] = $row['UNSPSC']; + $ane = str_replace("ane", "anae", $row['Title']); + $unspsc[$ane] = $row['UNSPSC']; + $airo = str_replace("airplane", "aeroplane", $row['Title']); + $unspsc[$airo] = $row['UNSPSC']; // some divergence from standard + $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']); + $unspsc[$forensicit] = $row['UNSPSC']; + $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']); + $unspsc[$powercable] = $row['UNSPSC']; $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']); $unspsc[$tobacco] = $row['UNSPSC']; $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); --- a/australian_federal_government_contract_spending.json +++ b/australian_federal_government_contract_spending.json @@ -1,125 +1,164 @@ { - "dataset":{ - "name":"australian_federal_government_contract_spending", - "label":"Australian Federal Government Contract Spending", - "description":"Spending by Australian Federal Government agencies on goods and services from 2007 onwards.", - "currency":"AUD", - "unique_keys":[ - "id" - ], - "temporal_granularity":"year" - }, - "mapping":{ - "category": { - "fields": [ { - "column": "category", - "datatype": "string", - "name": "label" - } ], - - "type": "classifier", - "description": "", - "taxonomy": "unspsc", - "label": "Contract Goods/Services Category" + "dataset": { + "languages": [ + "en" + ], + "currency": "AUD", + "name": "australian_federal_government_contract_spending", + "territories": [], + "default_time": null, + "description": "Spending by Australian Federal Government agencies on goods and services from 2007 onwards.", + "schema_version": "2011-12-07", + "label": "Australian Federal Government Contract Spending" }, - "from":{ - "fields":[ - { - "column":"agencyABN", - "datatype":"string", - "name":"id" - }, - { - "column":"agencyName", - "datatype":"string", - "name":"label" - } - ], - "type":"entity", - "description":"", - "label":"Government Agency Australian Business Number" - }, - "description":{ - "column":"description", - "datatype":"string", - "type":"value", - "description":"Contract as described by agency", - "label":"Contract Description" - }, - "to":{ - "fields":[ - { - "column":"supplierName", - "datatype":"string", - "name":"label" - }, - { - "column":"supplierID", - "datatype":"string", - "name":"id" - } - ], - "type":"entity", - "description":"Government Supplier", - "label":"Supplier Name" - }, - "currency":{ - "default_value":"AUD", - "description":"All entries in AUD, foreign transactions are converted at the time of their payment", - "column":"", - "label":"", - "datatype":"currency", - "type":"value" - }, - "amount":{ - "column":"value", - "datatype":"float", - "type":"value", - "description":"The total value of the contract including all variations/amendments/extensions", - "label":"Contract Value" - }, - "time":{ - "column":"contractStart", - "datatype":"date", - "type":"value", - "description":"Start of the contract period (goods or services being received)", - "label":"Contract Start Date" - }, - "id":{ -"column": "CNID", - "datatype": "string", - "name": "label", - "type": "value", - "description": "", - "label": "Contract Notice ID" - } - }, - -"views": [ + "mapping": { + "category": { + "description": "Contract Goods/Services Category", + "datatype": "string", + "label": "category", + "column": "cat1", + "type": "attribute", + "dimension": "category" + }, + "sourceurl": { + "description": "Source Document URL", + "datatype": "string", + "label": "sourceurl", + "column": "sourceurl", + "type": "attribute", + "dimension": "sourceurl" + }, + "from": { + "attributes": { + "name": { + "column": "agencyABN", + "datatype": "id", + "type": "id", + "description": null + }, + "label": { + "column": "agencyName", + "datatype": "string", + "type": "attribute", + "description": null + } + }, + "label": "Government Agency", + "type": "compound", + "dimension": "from", + "description": "Government Agency" + }, + "description": { + "description": null, + "datatype": "string", + "label": "description", + "column": "description", + "type": "attribute", + "dimension": "description" + }, + "publishdate": { + "description": null, + "format": null, + "datatype": "date", + "label": "publishDate", + "column": "publishDate", + "type": "date", + "dimension": "publishdate" + }, + "to": { + "attributes": { + "name": { + "column": "supplierid", + "datatype": "id", + "type": "id" + }, + "label": { + "column": "supplierName", + "datatype": "string", + "type": "attribute" + } + }, + "label": "Supplier Name", + "type": "compound", + "dimension": "to", + "description": "Government Supplier" + }, + "amount": { + "description": "The total value of the contract including all variations/amendments/extensions", + "datatype": "float", + "label": "value", + "column": "value", + "type": "measure", + "dimension": "amount" + }, + "cnid": { + "description": "Contract Notice ID", + "datatype": "string", + "label": "CNID", + "column": "CNID", + "key": true, + "type": "attribute", + "dimension": "cnid" + }, + "contractend": { + "description": "End of the contract period (goods or services being received)", + "format": null, + "datatype": "date", + "label": "contractEnd", + "column": "contractEnd", + "type": "date", + "dimension": "contractend" + }, + "time": { + "description": "Start of the contract period (goods or services being received)", + "format": null, + "datatype": "date", + "label": "contractStart", + "column": "contractStart", + "type": "date", + "dimension": "time" + } + }, + "views": [ { - "name": "default", - "entity": "dataset", - "label": "Default breakdown by Category", - "dimension": "dataset", - "breakdown": "category", - "filters" : {"name": "australian_federal_government_contract_spending" } - }, - { - "name":"default", - "entity": "classifier", - "label":"Breakdown of Spending Areas by Supplier Company", - "dimension": "category", - "breakdown": "to", - "filters": {"taxonomy": "unspsc"} - }, - { - "name":"default", - "entity": "classifier", - "label":"Breakdown of Spending Areas by Government Agency", - "dimension": "category", - "breakdown": "from", - "filters": {"taxonomy": "unspsc"} - } -] + "name": "default", + "label": "Breakdown by Category", + "cuts": {}, + "drilldown": "category", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "by_supplier", + "label": "Breakdown by Supplier", + "cuts": {}, + "drilldown": "to", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "by_agency", + "label": "Breakdown by Agency", + "cuts": {}, + "drilldown": "from", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "default", + "label": "Agency spending breakdown by supplier", + "cuts": {}, + "drilldown": "to", + "entity": "dimension", + "dimension": "from" + }, + { + "name": "default", + "label": "Supplier contracts breakdown by procuring agency", + "cuts": {}, + "drilldown": "from", + "entity": "dimension", + "dimension": "to" + } + ] } - --- a/displayBubbletree.php +++ b/displayBubbletree.php @@ -32,48 +32,48 @@ } $total = 0; -$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat +$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); $catsresult->execute(); foreach ($catsresult->fetchAll() as $row) { $cats[] = $row['cat']; } -$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat +$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL GROUP BY "categoryUNSPSC" order by cat ;'); $cattwosresult->execute(); foreach ($cattwosresult->fetchAll() as $tworow) { - $cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat']; + $cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat']; } -$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value +$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value FROM contractnotice WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL GROUP BY cat order by cat ;'); $catthreesresult->execute(); foreach ($catthreesresult->fetchAll() as $threerow) { - $catthrees[substr($threerow['cat'],0,2)][] = $threerow; + $catthrees[substr($threerow['cat'],0,4)][] = $threerow; } $nodes = Array(); foreach ($cats as $catNum) { $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); - $catName = $unspsc[$catNum . "0000000"] . $catNum; + $catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum; $subnodes = Array(); $catValue = 0; foreach ($cattwos[$catNum] as $twoCatNum) { - $subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum; + $subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum; $subsubnodes = Array(); - $subCatValue = 0; + $subCatValue = 1; $subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); foreach ($catthrees[$twoCatNum] as $threerow) { - $subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat']; + $subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat']; $subsubnodes[] = Array( "label" => $subsubcatName, "amount" => $threerow['value'], - //"color" => "#".$subCatColor + "color" => "#".$subCatColor ); $subCatValue += $threerow['value']; } --- a/exportData.csv.php +++ b/exportData.csv.php @@ -12,24 +12,16 @@ } $query = $conn->prepare(' -SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN", +SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN", EXTRACT(EPOCH FROM "publishDate") as "publishDate", EXTRACT(EPOCH FROM "contractStart") as "contractStart", EXTRACT(EPOCH FROM "contractEnd") as "contractEnd", -value,description,"procurementMethod",category,"categoryUNSPSC", -(substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "cat1", - (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "cat2", - (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text) as "cat3", - "supplierABN","supplierName", -( - case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, - +value,description,category, +"supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL -FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" +FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" where "childCN" is null' , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); -// "supplierCity","supplierPostcode","supplierCountry","contactPostcode", -// $query->execute(); $errors = $conn->errorInfo(); if ($errors[2] != "") { @@ -57,6 +49,9 @@ || $headers[$key] == "contractEnd") { $colvalue = date("Y-m-d", $colvalue); } + /* if ($headers[$key] == "CNID") { + $colvalue = str_replace("A","", $colvalue); +}*/ if ($headers[$key] == "cat1" || $headers[$key] == "cat2" || $headers[$key] == "cat3") { $colvalue = $unspsc[$colvalue]; --- /dev/null +++ b/robots.txt @@ -1,1 +1,3 @@ +User-agent: * +Disallow: /admin