projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Add beginnings of public contract ontology
Add beginnings of public contract ontology

file:a/admin/neo4jimporter/pom.xml -> file:b/admin/neo4jimporter/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.lambdacomplex.contractdashboard</groupId> <groupId>org.lambdacomplex.contractdashboard</groupId>
<artifactId>neo4jimporter</artifactId> <artifactId>neo4jimporter</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId> <artifactId>neo4j</artifactId>
<version>1.6.1</version> <version>1.8.RC1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>postgresql</groupId> <groupId>postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<version>9.0-801.jdbc4</version> <version>9.0-801.jdbc4</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>
file:a/admin/neo4jimporter/src/main/java/Importer.java -> file:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.io.ObjectInputStream.GetField;  
import java.math.BigInteger;  
import java.security.MessageDigest;  
import java.security.NoSuchAlgorithmException;  
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map;  
   
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndex;
import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.collection.MapUtil;
  import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
import org.neo4j.index.impl.lucene.*;  
   
public class Importer { public class Importer {
   
public static void main(String[] argv) { public static void main(String[] argv) {
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
labels.setCacheCapacity("Label", 100000); labels.setCacheCapacity("Label", 100000);
   
   
   
System.out.println("-------- PostgreSQL " System.out.println("-------- PostgreSQL "
+ "JDBC Connection Testing ------------"); + "JDBC Connection Testing ------------");
   
try { try {
   
Class.forName("org.postgresql.Driver"); Class.forName("org.postgresql.Driver");
   
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
   
System.out.println("Where is your PostgreSQL JDBC Driver? " System.out.println("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!"); + "Include in your library path!");
e.printStackTrace(); e.printStackTrace();
   
} }
   
System.out.println("PostgreSQL JDBC Driver Registered!"); System.out.println("PostgreSQL JDBC Driver Registered!");
   
Connection conn = null; Connection conn = null;
   
try { try {
   
conn = DriverManager.getConnection( conn = DriverManager.getConnection(
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
"postgres", "snmc"); "postgres", "snmc");
   
} catch (SQLException e) { } catch (SQLException e) {
   
System.out.println("Connection Failed! Check output console"); System.out.println("Connection Failed! Check output console");
e.printStackTrace(); e.printStackTrace();
   
} }
   
if (conn != null) { if (conn != null) {
System.out.println("You made it, take control your database now!"); System.out.println("You made it, take control your database now!");
} else { } else {
System.out.println("Failed to make connection!"); System.out.println("Failed to make connection!");
} }
try { try {
// Print all warnings // Print all warnings
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
System.out.println("SQL Warning:"); System.out.println("SQL Warning:");
System.out.println("State : " + warn.getSQLState()); System.out.println("State : " + warn.getSQLState());
System.out.println("Message: " + warn.getMessage()); System.out.println("Message: " + warn.getMessage());
System.out.println("Error : " + warn.getErrorCode()); System.out.println("Error : " + warn.getErrorCode());
} }
   
// Get a statement from the connection // Get a statement from the connection
Statement stmt = conn.createStatement(); Statement stmt = conn.createStatement();
   
// Execute the query // Execute the query
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" + "FROM public.contractnotice GROUP BY contractnotice.\"agencyName\", "
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "  
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
String previousAgency = ""; String previousAgency = "";
GraphDatabaseService gds = inserter.getGraphDbService(); GraphDatabaseService gds = inserter.getGraphDbService();
HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
   
// Loop through the result set // Loop through the result set
while (rs.next()) { while (rs.next()) {
long supplierID, agencyID; long supplierID, agencyID;
String supplierKey; String supplierKey;
if (agencyIDs.get(rs.getString("agencyName")) == null) { if (agencyIDs.get(rs.getString("agencyName")) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("agencyName")); myNode.setProperty("Label", rs.getString("agencyName"));
myNode.setProperty("type", "agency"); myNode.setProperty("type", "agency");
agencyIDs.put(rs.getString("agencyName"), myNode.getId()); agencyIDs.put(rs.getString("agencyName"), myNode.getId());
if (myNode.getId() % 100 == 0) { if (myNode.getId() % 100 == 0) {
System.out.println("Agency " + myNode.getId()); System.out.println("Agency " + myNode.getId());
} }
} }
agencyID = agencyIDs.get(rs.getString("agencyName")); agencyID = agencyIDs.get(rs.getString("agencyName"));
   
   
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
supplierKey = rs.getString("supplierABN"); supplierKey = rs.getString("supplierABN");
} else { } else {
supplierKey = rs.getString("supplierName"); supplierKey = rs.getString("supplierName");
} }
// inject some data // inject some data
if (supplierIDs.get(supplierKey) == null) { if (supplierIDs.get(supplierKey) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("supplierName")); myNode.setProperty("Label", rs.getString("supplierName"));
myNode.setProperty("type", "supplier"); myNode.setProperty("type", "supplier");
supplierIDs.put(supplierKey, myNode.getId()); supplierIDs.put(supplierKey, myNode.getId());
if (myNode.getId() % 1000 == 0) { if (myNode.getId() % 1000 == 0) {
System.out.println("Supplier " + myNode.getId()); System.out.println("Supplier " + myNode.getId());
} }
} }
supplierID = supplierIDs.get(supplierKey); supplierID = supplierIDs.get(supplierKey);
   
   
long rel = inserter.createRelationship(agencyID, supplierID, long rel = inserter.createRelationship(agencyID, supplierID,
DynamicRelationshipType.withName("KNOWS"), null); DynamicRelationshipType.withName("KNOWS"), null);
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
   
} }
// Close the result set, statement and the connection // Close the result set, statement and the connection
rs.close(); rs.close();
stmt.close(); stmt.close();
conn.close(); conn.close();
} catch (SQLException se) { } catch (SQLException se) {
System.out.println("SQL Exception:"); System.out.println("SQL Exception:");
   
// Loop through the SQL Exceptions // Loop through the SQL Exceptions
while (se != null) { while (se != null) {
System.out.println("State : " + se.getSQLState()); System.out.println("State : " + se.getSQLState());
System.out.println("Message: " + se.getMessage()); System.out.println("Message: " + se.getMessage());
System.out.println("Error : " + se.getErrorCode()); System.out.println("Error : " + se.getErrorCode());
   
se = se.getNextException(); se = se.getNextException();
} }
} }
//make the changes visible for reading, use this sparsely, requires IO! //make the changes visible for reading, use this sparsely, requires IO!
labels.flush(); labels.flush();
   
// Make sure to shut down the index provider // Make sure to shut down the index provider
indexProvider.shutdown(); indexProvider.shutdown();
inserter.shutdown(); inserter.shutdown();
} }
} }
file:a/displayAgency.php -> file:b/displayAgency.php
<?php <?php
include_once ("./lib/common.inc.php"); include_once ("./lib/common.inc.php");
if ($_REQUEST['agency']) { if ($_REQUEST['agency']) {
$agency = htmlentities(strip_tags($_REQUEST['agency'])); $agency = htmlentities(strip_tags($_REQUEST['agency']));
include_header($agency); include_header($agency);
echo '<center><h1>'.$agency.'</h1></center>'; echo '<center><h1>'.$agency.'</h1></center>';
// MethodCountGraph($agency); // MethodCountGraph($agency);
// CnCGraph($agency); // CnCGraph($agency);
// MethodValueGraph($agency); // MethodValueGraph($agency);
/* biggest contracts /* biggest contracts
spending by year spending by year
spending by industry/category spending by industry/category
spending by supplier spending by supplier
spread procurement methods (stacked bar graph) spread procurement methods (stacked bar graph)
+ percent consultancies + percent confidential (bar graph) + percent consultancies + percent confidential (bar graph)
Average value by procurement type Average value by procurement type
--- info --- info
website, procurement plan, annual reports website, procurement plan, annual reports
Breakdown of divisions/branches Breakdown of divisions/branches
Breakdown percentage,number,value by procurement type Breakdown percentage,number,value by procurement type
Histograph, overlaying number value reported per week over X years Histograph, overlaying number value reported per week over X years
Compliance statistics: amendments, delay in reporting average and number completely late */ Compliance statistics: amendments, delay in reporting average and number completely late */
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", $query = 'SELECT "CNID", "description", "value", "agencyName", "category",
"contractStart", "supplierName" "contractStart", "supplierName"
FROM contractnotice FROM contractnotice
WHERE "agencyName" = :agency WHERE "agencyName" like :agency
ORDER BY "value" DESC limit 100'; ORDER BY "value" DESC limit 100';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":agency", $agency); $query->bindParam(":agency", $agency);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['value']), 2); $value = number_format(doubleval($row['value']), 2);
echo ("<tr> echo ("<tr>
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td>
<td><b>{$row['description']}</b></a></td> <td><b>{$row['description']}</b></a></td>
<td>\$$value</td><td>{$row['agencyName']}</td> <td>\$$value</td><td>{$row['agencyName']}</td>
<td>{$row['contractStart']}</td> <td>{$row['contractStart']}</td>
<td>{$row['supplierName']}</td> <td>{$row['supplierName']}</td>
</tr>"); </tr>");
} }
echo "</table>"; echo "</table>";
} else { } else {
/* /*
split by portfolio split by portfolio
*/ */
include_header("Agencies"); include_header("Agencies");
agenciesGraph(); agenciesGraph();
$query = 'SELECT SUM("value"), "agencyName" $query = 'SELECT SUM("value"), "agencyName"
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
GROUP BY "agencyName" '; GROUP BY "agencyName" ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Agency</th> <th>Agency</th>
<th>Total Contracts Value</th> <th>Total Contracts Value</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row[0]), 2); $value = number_format(doubleval($row[0]), 2);
$agency = stripslashes($row[1]); $agency = stripslashes($row[1]);
echo ("<tr><td><b><a href=\"displayAgency.php?agency={$agency}\">{$agency}</a></b></td><td>\$$value</td></tr>\n"); echo ("<tr><td><b><a href=\"displayAgency.php?agency={$agency}\">{$agency}</a></b></td><td>\$$value</td></tr>\n");
} }
echo "</table>"; echo "</table>";
} }
include_footer(); include_footer();
?> ?>
   
file:a/displayCalendar.php -> file:b/displayCalendar.php
<?php <?php
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
include_header("Months and Years");  
if ($_REQUEST['month']) { if ($_REQUEST['month']) {
echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>";  
$monthParts = explode("-", $_REQUEST['month']); $monthParts = explode("-", $_REQUEST['month']);
  $year = $monthParts[1];
  $month = $monthParts[0];
  $monthName = date("F Y", mktime(0,0,0,$month,1,$year));
   
  include_header($monthName);
  echo "<center><h1>" . $monthName . "</h1></center>";
   
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName" $query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName"
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
AND extract(year from "contractStart") = :year AND extract(year from "contractStart") = :year
AND extract(month from "contractStart") = :month AND extract(month from "contractStart") = :month
ORDER BY value DESC'; ORDER BY value DESC';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":month", $monthParts[0]); $query->bindParam(":month", $monthParts[0]);
$query->bindParam(":year", $monthParts[1]); $query->bindParam(":year", $monthParts[1]);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
//MethodCountGraph($supplier); //MethodCountGraph($supplier);
//CnCGraph($supplier); //CnCGraph($supplier);
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>