From: Maxious Date: Fri, 23 Dec 2011 00:23:24 +0000 Subject: Migrate agencies to 2 tables X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=c3370a6d6f329795a5c095574962d8faa08b1504 --- Migrate agencies to 2 tables --- --- a/admin/linkAmendments.php +++ b/admin/linkAmendments.php @@ -2,7 +2,7 @@ include_once ("../lib/common.inc.php"); $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID" WHERE -c."parentCN" > 0 AND p."childCN" IS NULL '; +c."parentCN" IS NOT NULL AND p."childCN" IS NULL '; $query = $conn->prepare($query); $query->execute(); databaseError($conn->errorInfo()); @@ -28,9 +28,9 @@ FROM contractnotice GROUP BY "parentCN" HAVING COUNT(*) > 1 - AND "parentCN" != 0 + AND "parentCN" IS NOT NULL ) -AND "childCN" = 0 +AND "childCN" IS NULL GROUP BY "parentCN" having count(*) > 1'; $query = $conn->prepare($query); $query->execute(); --- a/admin/updateUNSPSC.php +++ b/admin/updateUNSPSC.php @@ -43,6 +43,10 @@ $unspsc[$tobacco] = $row['UNSPSC']; $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); $unspsc[$architect] = $row['UNSPSC']; + $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']); + $unspsc[$powercable] = $row['UNSPSC']; + $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']); + $unspsc[$architect] = $row['UNSPSC']; // some just plain wrong $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']); $unspsc[$noOilRigs] = $row['UNSPSC']; --- a/australian_federal_government_contract_spending.json +++ b/australian_federal_government_contract_spending.json @@ -1,125 +1,164 @@ { - "dataset":{ - "name":"australian_federal_government_contract_spending", - "label":"Australian Federal Government Contract Spending", - "description":"Spending by Australian Federal Government agencies on goods and services from 2007 onwards.", - "currency":"AUD", - "unique_keys":[ - "id" - ], - "temporal_granularity":"year" - }, - "mapping":{ - "category": { - "fields": [ { - "column": "category", - "datatype": "string", - "name": "label" - } ], - - "type": "classifier", - "description": "", - "taxonomy": "unspsc", - "label": "Contract Goods/Services Category" + "dataset": { + "languages": [ + "en" + ], + "currency": "AUD", + "name": "australian_federal_government_contract_spending", + "territories": [], + "default_time": null, + "description": "Spending by Australian Federal Government agencies on goods and services from 2007 onwards.", + "schema_version": "2011-12-07", + "label": "Australian Federal Government Contract Spending" }, - "from":{ - "fields":[ - { - "column":"agencyABN", - "datatype":"string", - "name":"id" - }, - { - "column":"agencyName", - "datatype":"string", - "name":"label" - } - ], - "type":"entity", - "description":"", - "label":"Government Agency Australian Business Number" - }, - "description":{ - "column":"description", - "datatype":"string", - "type":"value", - "description":"Contract as described by agency", - "label":"Contract Description" - }, - "to":{ - "fields":[ - { - "column":"supplierName", - "datatype":"string", - "name":"label" - }, - { - "column":"supplierID", - "datatype":"string", - "name":"id" - } - ], - "type":"entity", - "description":"Government Supplier", - "label":"Supplier Name" - }, - "currency":{ - "default_value":"AUD", - "description":"All entries in AUD, foreign transactions are converted at the time of their payment", - "column":"", - "label":"", - "datatype":"currency", - "type":"value" - }, - "amount":{ - "column":"value", - "datatype":"float", - "type":"value", - "description":"The total value of the contract including all variations/amendments/extensions", - "label":"Contract Value" - }, - "time":{ - "column":"contractStart", - "datatype":"date", - "type":"value", - "description":"Start of the contract period (goods or services being received)", - "label":"Contract Start Date" - }, - "id":{ -"column": "CNID", - "datatype": "string", - "name": "label", - "type": "value", - "description": "", - "label": "Contract Notice ID" - } - }, - -"views": [ + "mapping": { + "category": { + "description": "Contract Goods/Services Category", + "datatype": "string", + "label": "category", + "column": "cat1", + "type": "attribute", + "dimension": "category" + }, + "sourceurl": { + "description": "Source Document URL", + "datatype": "string", + "label": "sourceurl", + "column": "sourceurl", + "type": "attribute", + "dimension": "sourceurl" + }, + "from": { + "attributes": { + "name": { + "column": "agencyABN", + "datatype": "id", + "type": "id", + "description": null + }, + "label": { + "column": "agencyName", + "datatype": "string", + "type": "attribute", + "description": null + } + }, + "label": "Government Agency", + "type": "compound", + "dimension": "from", + "description": "Government Agency" + }, + "description": { + "description": null, + "datatype": "string", + "label": "description", + "column": "description", + "type": "attribute", + "dimension": "description" + }, + "publishdate": { + "description": null, + "format": null, + "datatype": "date", + "label": "publishDate", + "column": "publishDate", + "type": "date", + "dimension": "publishdate" + }, + "to": { + "attributes": { + "name": { + "column": "supplierid", + "datatype": "id", + "type": "id" + }, + "label": { + "column": "supplierName", + "datatype": "string", + "type": "attribute" + } + }, + "label": "Supplier Name", + "type": "compound", + "dimension": "to", + "description": "Government Supplier" + }, + "amount": { + "description": "The total value of the contract including all variations/amendments/extensions", + "datatype": "float", + "label": "value", + "column": "value", + "type": "measure", + "dimension": "amount" + }, + "cnid": { + "description": "Contract Notice ID", + "datatype": "string", + "label": "CNID", + "column": "CNID", + "key": true, + "type": "attribute", + "dimension": "cnid" + }, + "contractend": { + "description": "End of the contract period (goods or services being received)", + "format": null, + "datatype": "date", + "label": "contractEnd", + "column": "contractEnd", + "type": "date", + "dimension": "contractend" + }, + "time": { + "description": "Start of the contract period (goods or services being received)", + "format": null, + "datatype": "date", + "label": "contractStart", + "column": "contractStart", + "type": "date", + "dimension": "time" + } + }, + "views": [ { - "name": "default", - "entity": "dataset", - "label": "Default breakdown by Category", - "dimension": "dataset", - "breakdown": "category", - "filters" : {"name": "australian_federal_government_contract_spending" } - }, - { - "name":"default", - "entity": "classifier", - "label":"Breakdown of Spending Areas by Supplier Company", - "dimension": "category", - "breakdown": "to", - "filters": {"taxonomy": "unspsc"} - }, - { - "name":"default", - "entity": "classifier", - "label":"Breakdown of Spending Areas by Government Agency", - "dimension": "category", - "breakdown": "from", - "filters": {"taxonomy": "unspsc"} - } -] + "name": "default", + "label": "Breakdown by Category", + "cuts": {}, + "drilldown": "category", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "by_supplier", + "label": "Breakdown by Supplier", + "cuts": {}, + "drilldown": "to", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "by_agency", + "label": "Breakdown by Agency", + "cuts": {}, + "drilldown": "from", + "entity": "dataset", + "dimension": "dataset" + }, + { + "name": "default", + "label": "Agency spending breakdown by supplier", + "cuts": {}, + "drilldown": "to", + "entity": "dimension", + "dimension": "from" + }, + { + "name": "default", + "label": "Supplier contracts breakdown by procuring agency", + "cuts": {}, + "drilldown": "from", + "entity": "dimension", + "dimension": "to" + } + ] } - --- a/displayBubbletree.php +++ b/displayBubbletree.php @@ -32,48 +32,48 @@ } $total = 0; -$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 2 ) as cat +$catsresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL group by "categoryUNSPSC";;'); $catsresult->execute(); foreach ($catsresult->fetchAll() as $row) { $cats[] = $row['cat']; } -$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 3 ) as cat +$cattwosresult = $conn->prepare('SELECT distinct substr( "categoryUNSPSC"::text, 0, 5 ) as cat FROM contractnotice WHERE "categoryUNSPSC" IS NOT NULL GROUP BY "categoryUNSPSC" order by cat ;'); $cattwosresult->execute(); foreach ($cattwosresult->fetchAll() as $tworow) { - $cattwos[substr($tworow['cat'],0,1)][] = $tworow['cat']; + $cattwos[substr($tworow['cat'],0,2)][] = $tworow['cat']; } -$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 4 ) as cat , SUM( "value" ) as value +$catthreesresult = $conn->prepare('SELECT substr( "categoryUNSPSC"::text, 0, 7 ) as cat , SUM( "value" ) as value FROM contractnotice WHERE "childCN" is null and "categoryUNSPSC" IS NOT NULL GROUP BY cat order by cat ;'); $catthreesresult->execute(); foreach ($catthreesresult->fetchAll() as $threerow) { - $catthrees[substr($threerow['cat'],0,2)][] = $threerow; + $catthrees[substr($threerow['cat'],0,4)][] = $threerow; } $nodes = Array(); foreach ($cats as $catNum) { $catColor = $color->hsv2hex(Array($catNum/10, .7, abs(($catNum*(1/10))-.5) + .5)); - $catName = $unspsc[$catNum . "0000000"] . $catNum; + $catName = substr($unspsc[$catNum . "000000"],0,18) . $catNum; $subnodes = Array(); $catValue = 0; foreach ($cattwos[$catNum] as $twoCatNum) { - $subcatName = $unspsc[$twoCatNum. "000000"] .$twoCatNum; + $subcatName = $unspsc[$twoCatNum. "0000"] .$twoCatNum; $subsubnodes = Array(); - $subCatValue = 0; + $subCatValue = 1; $subCatColor = $color->hsv2hex(Array($catNum/10, rand(1,10)/10, abs(($catNum*(1/10))-.5) + .5)); foreach ($catthrees[$twoCatNum] as $threerow) { - $subsubcatName = $unspsc[$threerow['cat'] . "00000"] . $threerow['cat']; + $subsubcatName = $unspsc[$threerow['cat'] . "00"] . $threerow['cat']; $subsubnodes[] = Array( "label" => $subsubcatName, "amount" => $threerow['value'], - //"color" => "#".$subCatColor + "color" => "#".$subCatColor ); $subCatValue += $threerow['value']; } --- a/exportData.csv.php +++ b/exportData.csv.php @@ -3,17 +3,25 @@ include_once("./lib/common.inc.php"); setlocale(LC_CTYPE, 'C'); // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 + +$unspsc = Array(); +$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';'); +$unspscresult->execute(); +foreach ($unspscresult->fetchAll() as $row) { + $unspsc[$row['UNSPSC']] = $row['Title']; +} + $query = $conn->prepare(' -SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", - "supplierABN","supplierName", -( - case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, - +SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN", +EXTRACT(EPOCH FROM "publishDate") as "publishDate", +EXTRACT(EPOCH FROM "contractStart") as "contractStart", +EXTRACT(EPOCH FROM "contractEnd") as "contractEnd", +value,description,category, +"supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL -FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" is null' +FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" +where "childCN" is null' , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); -// "supplierCity","supplierPostcode","supplierCountry","contactPostcode", -// (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3") $query->execute(); $errors = $conn->errorInfo(); if ($errors[2] != "") { @@ -21,8 +29,8 @@ } $num_fields = $query->columnCount(); -$headers = array(); -for ($i = 0; $i < $num_fields; $i++) { +$headers = Array(); +for ($i = 0; $i < $num_fields; $i++) { // for each column in query, make a CSV header $meta = $query->getColumnMeta($i); $headers[] = $meta['name']; } @@ -41,6 +49,13 @@ || $headers[$key] == "contractEnd") { $colvalue = date("Y-m-d", $colvalue); } + /* if ($headers[$key] == "CNID") { + $colvalue = str_replace("A","", $colvalue); +}*/ + if ($headers[$key] == "cat1" || $headers[$key] == "cat2" + || $headers[$key] == "cat3") { + $colvalue = $unspsc[$colvalue]; + } } fputcsv($fp, array_values($row)); } --- /dev/null +++ b/robots.txt @@ -1,1 +1,3 @@ +User-agent: * +Disallow: /admin