projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Add beginnings of public contract ontology
Add beginnings of public contract ontology

file:a/admin/neo4jimporter/src/main/java/Importer.java -> file:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
   
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndex;
import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider; import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
   
public class Importer { public class Importer {
   
public static void main(String[] argv) { public static void main(String[] argv) {
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
labels.setCacheCapacity("Label", 100000); labels.setCacheCapacity("Label", 100000);
   
   
   
System.out.println("-------- PostgreSQL " System.out.println("-------- PostgreSQL "
+ "JDBC Connection Testing ------------"); + "JDBC Connection Testing ------------");
   
try { try {
   
Class.forName("org.postgresql.Driver"); Class.forName("org.postgresql.Driver");
   
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
   
System.out.println("Where is your PostgreSQL JDBC Driver? " System.out.println("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!"); + "Include in your library path!");
e.printStackTrace(); e.printStackTrace();
   
} }
   
System.out.println("PostgreSQL JDBC Driver Registered!"); System.out.println("PostgreSQL JDBC Driver Registered!");
   
Connection conn = null; Connection conn = null;
   
try { try {
   
conn = DriverManager.getConnection( conn = DriverManager.getConnection(
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
"postgres", "snmc"); "postgres", "snmc");
   
} catch (SQLException e) { } catch (SQLException e) {
   
System.out.println("Connection Failed! Check output console"); System.out.println("Connection Failed! Check output console");
e.printStackTrace(); e.printStackTrace();
   
} }
   
if (conn != null) { if (conn != null) {
System.out.println("You made it, take control your database now!"); System.out.println("You made it, take control your database now!");
} else { } else {
System.out.println("Failed to make connection!"); System.out.println("Failed to make connection!");
} }
try { try {
// Print all warnings // Print all warnings
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
System.out.println("SQL Warning:"); System.out.println("SQL Warning:");
System.out.println("State : " + warn.getSQLState()); System.out.println("State : " + warn.getSQLState());
System.out.println("Message: " + warn.getMessage()); System.out.println("Message: " + warn.getMessage());
System.out.println("Error : " + warn.getErrorCode()); System.out.println("Error : " + warn.getErrorCode());
} }
   
// Get a statement from the connection // Get a statement from the connection
Statement stmt = conn.createStatement(); Statement stmt = conn.createStatement();
   
// Execute the query // Execute the query
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" + "FROM public.contractnotice GROUP BY contractnotice.\"agencyName\", "
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "  
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
String previousAgency = ""; String previousAgency = "";
GraphDatabaseService gds = inserter.getGraphDbService(); GraphDatabaseService gds = inserter.getGraphDbService();
HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
   
// Loop through the result set // Loop through the result set
while (rs.next()) { while (rs.next()) {
long supplierID, agencyID; long supplierID, agencyID;
String supplierKey; String supplierKey;
if (agencyIDs.get(rs.getString("agencyName")) == null) { if (agencyIDs.get(rs.getString("agencyName")) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("agencyName")); myNode.setProperty("Label", rs.getString("agencyName"));
myNode.setProperty("type", "agency"); myNode.setProperty("type", "agency");
agencyIDs.put(rs.getString("agencyName"), myNode.getId()); agencyIDs.put(rs.getString("agencyName"), myNode.getId());
if (myNode.getId() % 100 == 0) { if (myNode.getId() % 100 == 0) {
System.out.println("Agency " + myNode.getId()); System.out.println("Agency " + myNode.getId());
} }
} }
agencyID = agencyIDs.get(rs.getString("agencyName")); agencyID = agencyIDs.get(rs.getString("agencyName"));
   
   
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
supplierKey = rs.getString("supplierABN"); supplierKey = rs.getString("supplierABN");
} else { } else {
supplierKey = rs.getString("supplierName"); supplierKey = rs.getString("supplierName");
} }
// inject some data // inject some data
if (supplierIDs.get(supplierKey) == null) { if (supplierIDs.get(supplierKey) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("supplierName")); myNode.setProperty("Label", rs.getString("supplierName"));
myNode.setProperty("type", "supplier"); myNode.setProperty("type", "supplier");
supplierIDs.put(supplierKey, myNode.getId()); supplierIDs.put(supplierKey, myNode.getId());
if (myNode.getId() % 1000 == 0) { if (myNode.getId() % 1000 == 0) {
System.out.println("Supplier " + myNode.getId()); System.out.println("Supplier " + myNode.getId());
} }
} }
supplierID = supplierIDs.get(supplierKey); supplierID = supplierIDs.get(supplierKey);
   
   
long rel = inserter.createRelationship(agencyID, supplierID, long rel = inserter.createRelationship(agencyID, supplierID,
DynamicRelationshipType.withName("KNOWS"), null); DynamicRelationshipType.withName("KNOWS"), null);
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
   
} }
// Close the result set, statement and the connection // Close the result set, statement and the connection
rs.close(); rs.close();
stmt.close(); stmt.close();
conn.close(); conn.close();
} catch (SQLException se) { } catch (SQLException se) {
System.out.println("SQL Exception:"); System.out.println("SQL Exception:");
   
// Loop through the SQL Exceptions // Loop through the SQL Exceptions
while (se != null) { while (se != null) {
System.out.println("State : " + se.getSQLState()); System.out.println("State : " + se.getSQLState());
System.out.println("Message: " + se.getMessage()); System.out.println("Message: " + se.getMessage());
System.out.println("Error : " + se.getErrorCode()); System.out.println("Error : " + se.getErrorCode());
   
se = se.getNextException(); se = se.getNextException();
} }
} }
//make the changes visible for reading, use this sparsely, requires IO! //make the changes visible for reading, use this sparsely, requires IO!
labels.flush(); labels.flush();
   
// Make sure to shut down the index provider // Make sure to shut down the index provider
indexProvider.shutdown(); indexProvider.shutdown();
inserter.shutdown(); inserter.shutdown();
} }
} }
file:a/displayContract.php -> file:b/displayContract.php
<?php <?php
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
$query = 'SELECT * $query = 'SELECT *
FROM contractnotice FROM contractnotice
WHERE "CNID" = :CNID LIMIT 1'; WHERE "CNID" = :CNID LIMIT 1';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":CNID", $_REQUEST['CNID']); $query->bindParam(":CNID", $_REQUEST['CNID']);
$query->execute(); $query->execute();
$contractResult = $query->fetch(PDO::FETCH_ASSOC); $contractResult = $query->fetch(PDO::FETCH_ASSOC);
if (!$contractResult) { if (!$contractResult) {
header("Status: 404 Not Found"); header("Status: 404 Not Found");
header("HTTP/1.0 404 Not Found"); header("HTTP/1.0 404 Not Found");
include_header("Contract Not Found"); include_header("Contract Not Found");
echo "<center><h1>No Contract Notice with that ID found</h1></center>"; echo "<center><h1>No Contract Notice with that ID found</h1></center>";
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
} else { } else {
$description = ucsmart($contractResult["description"]); $description = ucsmart($contractResult["description"]);
include_header($description); include_header($description);
echo '<center><h1>'.$description.'</h1></center>'; echo '<center><h1>'.$description.'</h1></center>
  <div about="http://contracts.disclosurelo.gs/displayContract.php?CNID='.$contractResult["CNID"].'" typeof="pc:Contract">';
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
foreach (array_filter($contractResult) as $key => $value) { foreach (array_filter($contractResult) as $key => $value) {
echo "<b>$key</b>&nbsp;"; echo "<b>$key</b>&nbsp;";
switch ($key) { switch ($key) {
  case "supplierABN":
  break;
case "supplierName": case "supplierName":
case "supplierABN": echo '
echo '<a href="displaySupplier.php?supplier=' . $contractResult['supplierABN'] . '-' . urlencode($contractResult['supplierName']) . '">' . $value . "</a>"; <a href="displaySupplier.php?supplier=' . $contractResult['supplierABN'] . '-' . urlencode($contractResult['supplierName']) . '">
  <span rel="pc:contractingAuthority" typeof="gr:BusinessEntity">
  <span property="gr:legalName">'
  . $contractResult['supplierName'] .
  '</span>'.
  (isset($contractResult['supplierABN']) && $contractResult['supplierABN'] != '' ? ' (ABN: <span property="br:officialNumber" lang="">'.$contractResult['supplierABN'].'</span>)' : '')
  .'</span></a>';
  break;
  case "CNID":
  echo '<span property="pc:referenceNumber" lang="">'.$value.'</span>';
  break;
  case "categoryUNSPSC":
  break;
  case "category":
  echo '<span rel="pc:mainObject" resource="[unspsc:'.$contractResult["categoryUNSPSC"].']">'.$value.'</span>';
break; break;
case "agencyName": case "agencyName":
echo '<a href="displayAgency.php?agency=' . urlencode($value) . '">' . $value . "</a>"; echo '<a href="displayAgency.php?agency=' . urlencode($value) . '">' . $value . "</a>";
break; break;
  case "contractStart":
  echo '<span property="pc:startDate" content="'.$value.'" datatype="xsd:date">'.$value.'</span>';
  break;
  case "contractEnd":
  echo '<span property="pc:endDate" content="'.$value.'" datatype="xsd:date">'.$value.'</span>';
  break;
case "value": case "value":
echo "$" . number_format(doubleval($value), 2); echo '<span rel="pc:agreedPrice" typeof="gr:PriceSpecification">
  <span property="gr:hasCurrency" content="AUD" lang="">$ </span>
  <span property="gr:hasCurrencyValue" content="'.doubleval($value).'" datatype="xsd:float">' . number_format(doubleval($value), 2)
  .'</span></span>';
break; break;
  case "description":
  echo '<span property="dcterms:title">'.$value.'</span>';
default: default:
echo str_replace(" ", "<br>", ucsmart($value)); echo str_replace(" ", "<br>", ucsmart($value));
} }
echo "<br>"; echo "<br>\n";
} }
echo '<br><a href="https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN' . $_REQUEST['CNID'] . '"> View original record @ tenders.gov.au</a><br>'; echo '<br><a href="https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN' . $_REQUEST['CNID'] . '"> View original record @ tenders.gov.au</a><br>
   
  </div>';
$query = 'SELECT * FROM heuristic_results where "CNID" = :CNID'; $query = 'SELECT * FROM heuristic_results where "CNID" = :CNID';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$agencyName = $input . '%'; $agencyName = $input . '%';
$query->bindParam(":CNID", $_REQUEST['CNID']); $query->bindParam(":CNID", $_REQUEST['CNID']);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
foreach ($query->fetchAll() as $r) { foreach ($query->fetchAll() as $r) {
echo "<b>{$r['heuristic_name']}</b>: {$r['heuristic_value']} (raw value: {$r['raw_value']}, mean: {$r['mean']}, stddev: {$r['stddev']})<br>"; echo "<b>{$r['heuristic_name']}</b>: {$r['heuristic_value']} (raw value: {$r['raw_value']}, mean: {$r['mean']}, stddev: {$r['stddev']})<br>";
} }
} }
include_footer(); include_footer();
?> ?>
   
file:a/lib/common.inc.php -> file:b/lib/common.inc.php
<?php <?php
date_default_timezone_set("Australia/ACT"); date_default_timezone_set("Australia/ACT");
error_reporting(E_ALL ^ E_NOTICE); error_reporting(E_ALL ^ E_NOTICE);
$conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost"); $conn = new PDO("pgsql:dbname=contractDashboard;user=postgres;password=snmc;host=localhost");
if (!$conn) { if (!$conn) {
die("A database error occurred.\n"); die("A database error occurred.\n");
} }
define('ROOT', pathinfo(__FILE__, PATHINFO_DIRNAME)); define('ROOT', pathinfo(__FILE__, PATHINFO_DIRNAME));
if (strstr($_SERVER['PHP_SELF'], "labs/")) { if (strstr($_SERVER['PHP_SELF'], "labs/")) {
$basePath = "../"; $basePath = "../";
} }
require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php'; require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php';
$openid = new LightOpenID($_SERVER['HTTP_HOST']); $openid = new LightOpenID($_SERVER['HTTP_HOST']);
function login() { function login() {
global $openid; global $openid;
if (!$openid->mode) { if (!$openid->mode) {
$openid->required = array('contact/email'); $openid->required = array('contact/email');
$openid->identity = 'https://www.google.com/accounts/o8/id'; $openid->identity = 'https://www.google.com/accounts/o8/id';
header('Location: ' . $openid->authUrl()); header('Location: ' . $openid->authUrl());
} }
} }
function auth() { function auth() {
global $openid; global $openid;
if ($_SESSION['authed'] == true) { if ($_SESSION['authed'] == true) {
return true; return true;
} }
if ($openid->mode) { if ($openid->mode) {
$attr = $openid->getAttributes(); $attr = $openid->getAttributes();
if ($attr['contact/email'] != 'maxious@gmail.com') { if ($attr['contact/email'] != 'maxious@gmail.com') {
die('Access Denied'); die('Access Denied');
} else { } else {
$_SESSION['authed'] = true; $_SESSION['authed'] = true;
} }
} else { } else {
login(); login();
} }
} }
// $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); // $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
function databaseError($errMsg) { function databaseError($errMsg) {
if ($errMsg[2] != "") { if ($errMsg[2] != "") {
echo '<div class="alert-message error">'; echo '<div class="alert-message error">';
die(print_r($errMsg, true)); die(print_r($errMsg, true));
echo "</div>"; echo "</div>";
} }
} }
function ucsmart($str) { function ucsmart($str) {
$shortWords = Array("The", "Pty", "Ltd", "Inc", "Red", "Oil", "A", "An", "And", "At", "For", "In" $shortWords = Array("The", "Pty", "Ltd", "Inc", "Red", "Oil", "A", "An", "And", "At", "For", "In"
, "Of", "On", "Or", "The", "To", "With"); , "Of", "On", "Or", "The", "To", "With");
$strArray = explode(" ", preg_replace("/(?<=(?<!:|’s)\W) $strArray = explode(" ", preg_replace("/(?<=(?<!:|’s)\W)
(A|An|And|At|For|In|Of|On|Or|The|To|With) (A|An|And|At|For|In|Of|On|Or|The|To|With)
(?=\W)/e", 'strtolower("$1")', ucwords(strtolower($str)))); (?=\W)/e", 'strtolower("$1")', ucwords(strtolower($str))));
foreach ($strArray as &$word) { foreach ($strArray as &$word) {
if (strlen($word) <= 4 && !in_array($word, $shortWords)) if (strlen($word) <= 4 && !in_array($word, $shortWords))
$word = strtoupper($word); $word = strtoupper($word);
} }
return implode(" ", $strArray); return implode(" ", $strArray);
} }
function percent($num_amount, $num_total) { function percent($num_amount, $num_total) {
$count1 = $num_amount / $num_total; $count1 = $num_amount / $num_total;
$count2 = $count1 * 100; $count2 = $count1 * 100;
$count = number_format($count2, 2); $count = number_format($count2, 2);
return $count; return $count;
} }