From: Alex Sadleir Date: Sun, 08 Dec 2013 04:22:54 +0000 Subject: remove amon, fix openid X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=bb352fd764348d6b0d8eb9f94f871209d5b801b2 --- remove amon, fix openid --- --- a/admin/data/CN806771.2cb256c016c609172a97a04741a278f8.xls +++ b/admin/data/CN806771.2cb256c016c609172a97a04741a278f8.xls @@ -1010,7 +1010,7 @@ ="Future Fund Management Agency" ="CN987101" ="CN987101-A2" 25-Oct-12 09:58 AM 19-Dec-12 10:44 AM ="published" 10-Oct-12 31-Mar-13 67000.00 " Contractor fee " ="FFMA0513" ="Temporary personnel services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "Contract extension" ="Ranstad Pty Ltd" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="28080275378" ="" ="" ="3000" ="Future Fund Management Agency" ="" ="CN1104112" 19-Dec-12 10:47 AM ="published" 26-Nov-12 01-Mar-13 15000.00 " Contractor fees " ="FFMA0527" ="Temporary personnel services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Randstad" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="28080275378" ="" ="" ="3000" ="Australian Securities and Investments Commission" ="" ="CN1104152" 19-Dec-12 09:50 AM ="published" 12-Apr-12 30-Jun-13 178970.00 " Vetting " ="2854" ="Security surveillance and detection" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Premier Protection Agency" ="PO Box 3401" ="Liverpool" ="2170" ="AUSTRALIA" ="No" ="37078491826" ="" ="" ="2000" -="Department of Defence" ="" ="CN1104192" 19-Dec-12 10:01 AM ="published" 22-Jul-09 30-Sep-09 26180.00 " MRH 90 PROJECT NET ACTIONS SCRIPT PROGRAMMER " ="4500714409" ="Computer services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="DESIGN AND BUILD THIS" ="14 LARKEN AVE" ="BAULKHAM HILLS" ="2153" ="AUSTRALIA" ="No" ="93223025707" ="ARMY Headquarters" ="Army Group" ="2000" +="Department of Defence" ="" ="CN1104192" 19-Dec-12 10:01 AM ="published" 22-Jul-09 30-Sep-09 26180.00 " MRH 90 PROJECT NET ACTIONS SCRIPT PROGRAMMER " ="4500714409" ="Computer services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="DESIGN AND BUILD THIS" ="" ="BAULKHAM HILLS" ="2153" ="AUSTRALIA" ="No" ="93223025707" ="ARMY Headquarters" ="Army Group" ="2000" ="Department of Industry, Innovation, Science, Research and Tertiary Education" ="CN1065821" ="CN1065821-A1" 03-Dec-12 02:22 PM 19-Dec-12 10:06 AM ="published" 07-Aug-12 30-Jun-13 13491.50 " HTML Conversion of 2011-12 Annual Report " ="CONTRACTS-004828-0" ="Internet services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "Increase contract value" ="SWELL DESIGN GROUP" ="" ="MANUKA" ="2603" ="AUSTRALIA" ="No" ="95060811725" ="" ="" ="2601" ="Defence Materiel Organisation" ="" ="CN1104232" 19-Dec-12 10:06 AM ="published" 19-Dec-12 21-May-13 87747.00 " purchase of test sets and shipping crates " ="NC1YY9" ="Aerospace systems and components and equipment" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="astrite pty ltd" ="5 jarrah court" ="hoppers crossing" ="3029" ="AUSTRALIA" ="No" ="18756753773" ="" ="" ="3027" ="Future Fund Management Agency" ="" ="CN1104262" 19-Dec-12 10:46 AM ="published" 30-Nov-12 30-Nov-12 31680.00 " Personnel Recruitment " ="FFMA0526" ="Personnel recruitment" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Mahlad Recruitment Pty Lts" ="" ="Melbourne" ="3000" ="AUSTRALIA" ="No" ="46633561134" ="" ="" ="3000" @@ -1847,4 +1847,5 @@ ="Office of the Director of Public Prosecutions" ="" ="CN1112292" 21-Dec-12 06:59 PM ="published" 01-Jan-13 31-Oct-13 120000.00 " Estimated value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Probe Group Pty Ltd" ="" ="Melbourne" ="3000" ="AUSTRALIA" ="No" ="90770747494" ="" ="" ="2601" ="Office of the Director of Public Prosecutions" ="" ="CN1112302" 21-Dec-12 07:02 PM ="published" 01-Jan-13 31-Oct-13 210000.00 " Estimated Value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Statewide Mercantile Services Pty Ltd" ="" ="West Melbourne" ="3000" ="AUSTRALIA" ="No" ="77075475544" ="" ="" ="2601" ="Office of the Director of Public Prosecutions" ="" ="CN1112312" 21-Dec-12 07:04 PM ="published" 01-Jan-13 31-Oct-13 205000.00 " Estimated value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Wise McGrath" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="14096930568" ="" ="" ="2601" -="Australian Law Reform Commission" ="" ="CN1014921" 17-Dec-12 02:20 PM ="published" 01-Nov-12 02-Nov-15 12000.00 " Office supplies and stationary " ="ALRC 02" ="Stationery" ="Open tender" ="FIN11/FMG010" ="SON473022" ="No" ="" ="No" ="" ="No" ="" "" ="OfficeMax Australia Limited" ="20 Worth St" ="Chullora" ="2190" ="AUSTRALIA" ="No" ="26064777224" ="Australian Law Reform Commission" ="Attorney-General's Department" ="2001" +="Australian Law Reform Commission" ="" ="CN1014921" 17-Dec-12 02:20 PM ="published" 01-Nov-12 02-Nov-15 12000.00 " Office supplies and stationary " ="ALRC 02" ="Stationery" ="Open tender" ="FIN11/FMG010" ="SON473022" ="No" ="" ="No" ="" ="No" ="" "" ="OfficeMax Australia Limited" ="20 Worth St" ="Chullora" ="2190" ="AUSTRALIA" ="No" ="26064777224" ="Australian Law Reform Commission" ="Attorney-General's Department" ="2001" + --- a/admin/import.php +++ b/admin/import.php @@ -86,7 +86,8 @@ "Agency Postcode" => "contactPostcode", "" => "" ); - + + $headers; while (($data = fgetcsv($handle, 1000, "\t")) !== false) { $num = count($data); @@ -171,7 +172,7 @@ } ksort($files); foreach ($files as $date => $fname) { - echo "$fname " . filesize($path . $fname) . " " . $date . "
"; + echo "$fname " . filesize($path . $fname) . " " . $date . "
"; } } else { $success = 0; --- a/admin/importdatagov.php +++ /dev/null @@ -1,216 +1,1 @@ - $f) { - $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; -} -$contractNoticeInsertQ.= ");"; -$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); - -function processFile($fpath) { - global $conn, $contractNoticeFields, $contractNoticeInsertQ; - $row = 1; - $success = 0; - ini_set('auto_detect_line_endings',TRUE); - $handle = fopen($fpath, "r"); - //"t" mode string translates windows line breaks to unix - $datamapping0712 = array( - - "Agency Name" => "agencyName", - "Parent Contract ID" => "parentCN", - "Contract ID" => "CNID", - "Publish Date" => "publishDate", - "Amendment Date" => "amendDate", - "Start Date" => "contractStart", - "End Date" => "contractEnd", - "Value" => "value", - "Description" => "description", - "Agency Ref ID" => "agencyID", - "UNSPSC Code" => "categoryUNSPSC", - "Title" => "category", - "Procurement Method" => "procurementMethod", - "ATM ID" => "atmID", - "SON ID" => "SONID", - "Confidentiality Contract Flag" => "confidentialityContract", - "Confidentiality Contract Reason" => "confidentialityContractReason", - "Confidentiality Outputs Flag" => "confidentialityOutputs", - "Confidentiality Outputs Reason" => "confidentialityOutputsReason", - "Consultancy Flag" => "consultancy", - "Consultancy Reason" => "consultancyReason", - "Amendment Reason" => "amendmentReason", - "Supplier Name" => "supplierName", - "Supplier Address" => "supplierAddress", - "Supplier Suburb" => "supplierCity", - "Supplier Postcode" => "supplierPostcode", - "Supplier Country" => "supplierCountry", - "Supplier ABN Exempt" => "supplierABNExempt", - "ABN" => "supplierABN", - "Contact Name" => "", - "Contact Phone" => "", - "Branch" => "contactBranch", - "Division" => "contactDivision", - "Office Postcode" => "contactPostcode", - - ); - - - while (($data = fgetcsv($handle, 10000)) !== false) { - //print_r($data); - $num = count($data); - if ($row == 1) { - $headers = $data; - } elseif ($row > 1) { - if ($num > count($datamapping0712)) { - die("Error in data import; data mapping fields out of bounds or changed $num > ".count($datamapping0712)."
" . $fpath . print_r($data)); - } - $contractNoticeInsert = Array(); - $supplierInsert = Array(); - $agencyInsert = Array(); - $contractNoticeInsert[] = $fpath; - $keys = array_keys($datamapping0712); - for ($c = 0; $c < $num; $c++) { - $data[$c] = trim($data[$c], "="); - $data[$c] = trim($data[$c], "\""); - if (in_array(($datamapping0712[$headers[$c]]), $contractNoticeFields)) { - if (($datamapping0712[$headers[$c]]) == "parentCN" || ($datamapping0712[$headers[$c]]) == "CNID") { - $data[$c] = substr($data[$c], 2); // take off the "CN" prefix - if ($data[$c] > 0 && $data[$c] != '0') { - $contractNoticeInsert[] = $data[$c]; - } else { - $contractNoticeInsert[] = null; - } - } elseif (($datamapping0712[$headers[$c]]) == "supplierABN") { - if ($data[$c] > 0 && $data[$c] != '0') { - $contractNoticeInsert[] = $data[$c]; - } else { - $contractNoticeInsert[] = null; - } - } elseif (($datamapping0712[$headers[$c]]) == "amendDate" || ($datamapping0712[$headers[$c]]) == "publishDate" || ($datamapping0712[$headers[$c]]) == "contractStart" || ($datamapping0712[$headers[$c]]) == "contractEnd") { - $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c])); - } else { - if (strstr("\" =", $data[$c] > 0)) { - die("Invalid Description field" . $contractNoticeInsert); - } - $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c])); - - $contractNoticeInsert[] = $colvalue; - } - } - } - flush(); - //print_r($contractNoticeInsert); - $contractNoticeInsertQ->execute($contractNoticeInsert); - $errors = $conn->errorInfo(); - if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) { - // echo "dupe
"; - } elseif ($errors[1] == 0) { - $success++; - } else { - foreach ($contractNoticeFields as $key => $cnf) { - echo var_dump($contractNoticeInsert[$key]) . $cnf . "
"; - } - echo $data[2] . " failed CN insert.
" . print_r($errors, true) . "
row $row

\n"; - } - - flush(); - //echo "
\n"; - } - $row++; - } - fclose($handle); - $contractNoticeInsertQ->closeCursor(); - - return $success; -} - -$path = 'datagovdata/'; -if ($_REQUEST["fname"] == "" && $argv[1] == "") { - echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list
"; - $dhandle = opendir($path); - // define an array to hold the files - $files = array(); - if ($dhandle) { - // loop through all of the files - while (false !== ($fname = readdir($dhandle))) { - if (($fname != '.') && ($fname != '..')) { - $files[date("c", filemtime($path . $fname)) . md5($fname)] = $fname; - } - } - } - ksort($files); - foreach ($files as $date => $fname) { - echo "$fname " . filesize($path . $fname) . " " . $date . "
"; - } -} else { - $success = 0; - $fname = $_REQUEST["fname"]; - if ($argv[1] != "") $fname = $argv[1]; - echo " ============== $fname ==============
"; - flush(); - $success+= processFile($path . $fname, "contractnotice"); - $success+= processFile($path . $fname, "agency"); - $success+= processFile($path . $fname, "supplier"); - echo "
$success records successfully created"; - - flush(); - // run post import data processing - // - if ($success > 0) { -$conn->exec("update datasets set \"lastUpdated\" = NOW() where title = 'Contract Notices'"); - echo "link amend
"; - include ("linkAmendments.php"); - echo "update UNSPSC
"; - include ("updateUNSPSC.php"); - } -// cn - -// agency -//include ("setAgencyStatus.php"); -//include ("setAgencyURLABN.php"); - -} -?> - --- a/admin/neo4jimporter/pom.xml +++ b/admin/neo4jimporter/pom.xml @@ -7,13 +7,12 @@ org.neo4j neo4j - 2.0.0-M03 + 1.8.RC1 postgresql postgresql - 9.1-901.jdbc4 + 9.0-801.jdbc4 - --- a/admin/neo4jimporter/src/main/java/Importer.java +++ b/admin/neo4jimporter/src/main/java/Importer.java @@ -5,34 +5,25 @@ import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; +import java.util.HashMap; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.Writer; -import java.util.HashMap; -import java.util.Map; - -import org.neo4j.graphdb.Direction; -import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.index.BatchInserterIndex; +import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.helpers.collection.MapUtil; -import org.neo4j.unsafe.batchinsert.BatchInserter; -import org.neo4j.unsafe.batchinsert.BatchInserters; +import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider; +import org.neo4j.kernel.impl.batchinsert.BatchInserter; +import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; public class Importer { public static void main(String[] argv) { -Map config = new HashMap(); -config.put( "neostore.nodestore.db.mapped_memory", "90M" ); -BatchInserter inserter = BatchInserters.inserter("target/batchinserter-example-config", config ); - //BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); - //BatchInserterIndex names = indexProvider.nodeIndex("names", MapUtil.stringMap("type", "exact")); - //names.setCacheCapacity("name", 100000); + BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); + BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); + BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); + labels.setCacheCapacity("Label", 100000); @@ -87,56 +78,52 @@ // Execute the query ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " - + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END) as supplierID , max(contractnotice.\"supplierName\") as \"supplierName\",sum(value) as sum " + + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " + "FROM public.contractnotice GROUP BY contractnotice.\"agencyName\", " - + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END)"); + + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); + String previousAgency = ""; + GraphDatabaseService gds = inserter.getGraphDbService(); HashMap supplierIDs = new HashMap(); HashMap agencyIDs = new HashMap(); - -Label agencyLabel = DynamicLabel.label( "Agency" ); -inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" ); -Label supplierLabel = DynamicLabel.label( "Supplier" ); -inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" ); // Loop through the result set while (rs.next()) { long supplierID, agencyID; String supplierKey; if (agencyIDs.get(rs.getString("agencyName")) == null) { - Map properties = new HashMap(); - properties.put("name", rs.getString("agencyName")); - properties.put("type", rs.getString("agency")); - agencyID = inserter.createNode(properties, agencyLabel); - agencyIDs.put(rs.getString("agencyName"), agencyID); - if (agencyID % 10 == 0) { - System.out.println("Agency " + agencyID); + Node myNode = gds.createNode(); + myNode.setProperty("Label", rs.getString("agencyName")); + myNode.setProperty("type", "agency"); + agencyIDs.put(rs.getString("agencyName"), myNode.getId()); + if (myNode.getId() % 100 == 0) { + System.out.println("Agency " + myNode.getId()); } } agencyID = agencyIDs.get(rs.getString("agencyName")); + if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { + supplierKey = rs.getString("supplierABN"); + } else { + supplierKey = rs.getString("supplierName"); + } // inject some data - if (supplierIDs.get(rs.getString("supplierID")) == null) { - Map properties = new HashMap(); - properties.put("name", rs.getString("supplierName")); - properties.put("type", rs.getString("supplier")); - supplierID = inserter.createNode(properties, supplierLabel); - supplierIDs.put(rs.getString("supplierID"), supplierID); - if (supplierID % 1000 == 0) { - System.out.println("Supplier " + supplierID); + if (supplierIDs.get(supplierKey) == null) { + Node myNode = gds.createNode(); + myNode.setProperty("Label", rs.getString("supplierName")); + myNode.setProperty("type", "supplier"); + supplierIDs.put(supplierKey, myNode.getId()); + if (myNode.getId() % 1000 == 0) { + System.out.println("Supplier " + myNode.getId()); } } - supplierID = supplierIDs.get(rs.getString("supplierID")); + supplierID = supplierIDs.get(supplierKey); -// To set properties on the relationship, use a properties map -// instead of null as the last parameter. -Map properties = new HashMap(); -properties.put( "value", rs.getDouble("sum")); - inserter.createRelationship(agencyID, supplierID, - DynamicRelationshipType.withName("PAYS"), properties); - inserter.createRelationship(supplierID, agencyID, - DynamicRelationshipType.withName("PAID_BY"), properties); + long rel = inserter.createRelationship(agencyID, supplierID, + DynamicRelationshipType.withName("KNOWS"), null); + inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); + } // Close the result set, statement and the connection rs.close(); @@ -155,11 +142,10 @@ } } //make the changes visible for reading, use this sparsely, requires IO! -// names.flush(); + labels.flush(); // Make sure to shut down the index provider -// indexProvider.shutdown(); + indexProvider.shutdown(); inserter.shutdown(); } -} - +} --- a/admin/partialdata/import.php +++ b/admin/partialdata/import.php @@ -12,7 +12,6 @@ $handle = fopen($fpath, "r"); //"t" mode string translates windows line breaks to unix $datamapping0507 = array( - "Agency" => "agencyName", "CN ID" => "CNID", "Publish Date" => "publishDate", @@ -138,7 +137,7 @@ // loop through all of the files while (false !== ($fname = readdir($dhandle))) { if (($fname != '.') && ($fname != '..') && (!isset($_REQUEST["filter"]) || strpos($fname,$_REQUEST["filter"]) != false)) { - echo "$fname " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "
"; + echo "$fname " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "
"; processFile($path . $fname, "contractnotice"); } } --- a/admin/partialdata/importamendments.php +++ b/admin/partialdata/importamendments.php @@ -310,7 +310,7 @@ // loop through all of the files while (false !== ($fname = readdir($dhandle))) { if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) { - echo "$fname " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "
"; + echo "$fname " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "
"; processFile($path . $fname, "contractnotice"); } } --- a/exportOverview.csv.php +++ b/exportOverview.csv.php @@ -11,11 +11,10 @@ $unspsc[$row['UNSPSC']] = $row['Title']; } -//(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url $query = $conn->prepare(' -SELECT "CNID" as id, description as text -from "contractnotice" -where "childCN" is null limit 10000' +SELECT "CNID" as uid, description as text, +(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url from "contractnotice" +where "childCN" is null' , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); $query->execute(); $errors = $conn->errorInfo(); --- a/lib/common.inc.php +++ b/lib/common.inc.php @@ -14,14 +14,6 @@ if (strstr($_SERVER['PHP_SELF'], "labs/") || strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "heuristics/")) { $basePath = "../"; } -if ($_SERVER['HTTP_HOST'] != "localhost") { -require $basePath."lib/amon-php/amon.php"; -Amon::config(array('address'=> 'http://127.0.0.1:2464', - 'protocol' => 'http', - 'secret_key' => "JBcSUdFOi5lK0vCjLjbHDpQamcBnRA4iV7QLaTADeDQ")); -Amon::setup_exception_handler(); -} - require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php'; $openid = new LightOpenID($_SERVER['HTTP_HOST']); // you have to open the session to be able to modify or remove it @@ -314,3 +306,6 @@ include ("graphs.inc.php"); + + + --- a/lib/openid.php +++ b/lib/openid.php @@ -81,7 +81,7 @@ $this->trustRoot = (strpos($host, '://') ? $host : 'https://' . $host); } - if(($host_end = strpos($this->trustRoot, '/', 8)) !== false) { + if(($host_end = @strpos($this->trustRoot, '/', 8)) !== false) { $this->trustRoot = substr($this->trustRoot, 0, $host_end); }