projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Descriptive title tags
Descriptive title tags

file:a/admin/neo4jimporter/pom.xml -> file:b/admin/neo4jimporter/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.lambdacomplex.contractdashboard</groupId> <groupId>org.lambdacomplex.contractdashboard</groupId>
<artifactId>neo4jimporter</artifactId> <artifactId>neo4jimporter</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId> <artifactId>neo4j</artifactId>
<version>1.6.1</version> <version>1.8.RC1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>postgresql</groupId> <groupId>postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<version>9.0-801.jdbc4</version> <version>9.0-801.jdbc4</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>
file:a/admin/neo4jimporter/src/main/java/Importer.java -> file:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.io.ObjectInputStream.GetField;  
import java.math.BigInteger;  
import java.security.MessageDigest;  
import java.security.NoSuchAlgorithmException;  
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map;  
   
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndex;
import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.collection.MapUtil;
  import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
import org.neo4j.index.impl.lucene.*;  
   
public class Importer { public class Importer {
   
public static void main(String[] argv) { public static void main(String[] argv) {
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
labels.setCacheCapacity("Label", 100000); labels.setCacheCapacity("Label", 100000);
   
   
   
System.out.println("-------- PostgreSQL " System.out.println("-------- PostgreSQL "
+ "JDBC Connection Testing ------------"); + "JDBC Connection Testing ------------");
   
try { try {
   
Class.forName("org.postgresql.Driver"); Class.forName("org.postgresql.Driver");
   
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
   
System.out.println("Where is your PostgreSQL JDBC Driver? " System.out.println("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!"); + "Include in your library path!");
e.printStackTrace(); e.printStackTrace();
   
} }
   
System.out.println("PostgreSQL JDBC Driver Registered!"); System.out.println("PostgreSQL JDBC Driver Registered!");
   
Connection conn = null; Connection conn = null;
   
try { try {
   
conn = DriverManager.getConnection( conn = DriverManager.getConnection(
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
"postgres", "snmc"); "postgres", "snmc");
   
} catch (SQLException e) { } catch (SQLException e) {
   
System.out.println("Connection Failed! Check output console"); System.out.println("Connection Failed! Check output console");
e.printStackTrace(); e.printStackTrace();
   
} }
   
if (conn != null) { if (conn != null) {
System.out.println("You made it, take control your database now!"); System.out.println("You made it, take control your database now!");
} else { } else {
System.out.println("Failed to make connection!"); System.out.println("Failed to make connection!");
} }
try { try {
// Print all warnings // Print all warnings
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
System.out.println("SQL Warning:"); System.out.println("SQL Warning:");
System.out.println("State : " + warn.getSQLState()); System.out.println("State : " + warn.getSQLState());
System.out.println("Message: " + warn.getMessage()); System.out.println("Message: " + warn.getMessage());
System.out.println("Error : " + warn.getErrorCode()); System.out.println("Error : " + warn.getErrorCode());
} }
   
// Get a statement from the connection // Get a statement from the connection
Statement stmt = conn.createStatement(); Statement stmt = conn.createStatement();
   
// Execute the query // Execute the query
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", " + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
String previousAgency = ""; String previousAgency = "";
GraphDatabaseService gds = inserter.getGraphDbService(); GraphDatabaseService gds = inserter.getGraphDbService();
HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
   
// Loop through the result set // Loop through the result set
while (rs.next()) { while (rs.next()) {
long supplierID, agencyID; long supplierID, agencyID;
String supplierKey; String supplierKey;
if (agencyIDs.get(rs.getString("agencyName")) == null) { if (agencyIDs.get(rs.getString("agencyName")) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("agencyName")); myNode.setProperty("Label", rs.getString("agencyName"));
myNode.setProperty("type", "agency"); myNode.setProperty("type", "agency");
agencyIDs.put(rs.getString("agencyName"), myNode.getId()); agencyIDs.put(rs.getString("agencyName"), myNode.getId());
if (myNode.getId() % 100 == 0) { if (myNode.getId() % 100 == 0) {
System.out.println("Agency " + myNode.getId()); System.out.println("Agency " + myNode.getId());
} }
} }
agencyID = agencyIDs.get(rs.getString("agencyName")); agencyID = agencyIDs.get(rs.getString("agencyName"));
   
   
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
supplierKey = rs.getString("supplierABN"); supplierKey = rs.getString("supplierABN");
} else { } else {
supplierKey = rs.getString("supplierName"); supplierKey = rs.getString("supplierName");
} }
// inject some data // inject some data
if (supplierIDs.get(supplierKey) == null) { if (supplierIDs.get(supplierKey) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("supplierName")); myNode.setProperty("Label", rs.getString("supplierName"));
myNode.setProperty("type", "supplier"); myNode.setProperty("type", "supplier");
supplierIDs.put(supplierKey, myNode.getId()); supplierIDs.put(supplierKey, myNode.getId());
if (myNode.getId() % 1000 == 0) { if (myNode.getId() % 1000 == 0) {
System.out.println("Supplier " + myNode.getId()); System.out.println("Supplier " + myNode.getId());
} }
} }
supplierID = supplierIDs.get(supplierKey); supplierID = supplierIDs.get(supplierKey);
   
   
long rel = inserter.createRelationship(agencyID, supplierID, long rel = inserter.createRelationship(agencyID, supplierID,
DynamicRelationshipType.withName("KNOWS"), null); DynamicRelationshipType.withName("KNOWS"), null);
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
   
} }
// Close the result set, statement and the connection // Close the result set, statement and the connection
rs.close(); rs.close();
stmt.close(); stmt.close();
conn.close(); conn.close();
} catch (SQLException se) { } catch (SQLException se) {
System.out.println("SQL Exception:"); System.out.println("SQL Exception:");
   
// Loop through the SQL Exceptions // Loop through the SQL Exceptions
while (se != null) { while (se != null) {
System.out.println("State : " + se.getSQLState()); System.out.println("State : " + se.getSQLState());
System.out.println("Message: " + se.getMessage()); System.out.println("Message: " + se.getMessage());
System.out.println("Error : " + se.getErrorCode()); System.out.println("Error : " + se.getErrorCode());
   
se = se.getNextException(); se = se.getNextException();
} }
} }
//make the changes visible for reading, use this sparsely, requires IO! //make the changes visible for reading, use this sparsely, requires IO!
labels.flush(); labels.flush();
   
// Make sure to shut down the index provider // Make sure to shut down the index provider
indexProvider.shutdown(); indexProvider.shutdown();
inserter.shutdown(); inserter.shutdown();
} }
} }
file:a/displayCalendar.php -> file:b/displayCalendar.php
<?php <?php
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
include_header("Months and Years");  
if ($_REQUEST['month']) { if ($_REQUEST['month']) {
echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>";  
$monthParts = explode("-", $_REQUEST['month']); $monthParts = explode("-", $_REQUEST['month']);
  $year = $monthParts[1];
  $month = $monthParts[0];
  $monthName = date("F Y", mktime(0,0,0,$month,1,$year));
   
  include_header($monthName);
  echo "<center><h1>" . $monthName . "</h1></center>";
   
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName" $query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName"
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
AND extract(year from "contractStart") = :year AND extract(year from "contractStart") = :year
AND extract(month from "contractStart") = :month AND extract(month from "contractStart") = :month
ORDER BY value DESC'; ORDER BY value DESC';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":month", $monthParts[0]); $query->bindParam(":month", $monthParts[0]);
$query->bindParam(":year", $monthParts[1]); $query->bindParam(":year", $monthParts[1]);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
//MethodCountGraph($supplier); //MethodCountGraph($supplier);
//CnCGraph($supplier); //CnCGraph($supplier);
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['value']), 2); $value = number_format(doubleval($row['value']), 2);
echo ("<tr> echo ("<tr>
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td>
<td><b>{$row['description']}</b></a></td> <td><b>{$row['description']}</b></a></td>
<td>\$$value</td><td>{$row['agencyName']}</td> <td>\$$value</td><td>{$row['agencyName']}</td>
<td>{$row['contractStart']}</td> <td>{$row['contractStart']}</td>
<td>{$row['supplierName']}</td> <td>{$row['supplierName']}</td>
</tr>"); </tr>");
} }
echo "</table>"; echo "</table>";
} else { } else {
/* /*
split by year split by year
todo: todo:
Year/Month drilldown - largest contracts, agencies, suppliers Year/Month drilldown - largest contracts, agencies, suppliers
count per month count per month
big picture graphs? */ big picture graphs? */
  include_header("Months and Years");
ContractStartingGraph(); ContractStartingGraph();
ContractPublishedGraph(); ContractPublishedGraph();
$query = 'SELECT extract(year from "contractStart"), extract(month from "contractStart"), $query = 'SELECT extract(year from "contractStart"), extract(month from "contractStart"),
SUM(value) as val, count(1) as count FROM contractnotice WHERE "childCN" is null GROUP BY extract(year from "contractStart"), extract(month from "contractStart") ORDER BY extract(year from "contractStart"), extract(month from "contractStart") '; SUM(value) as val, count(1) as count FROM contractnotice WHERE "childCN" is null GROUP BY extract(year from "contractStart"), extract(month from "contractStart") ORDER BY extract(year from "contractStart"), extract(month from "contractStart") ';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Month/Year</th> <th>Month/Year</th>
<th>Total Contracts Value</th> <th>Total Contracts Value</th>
<th>Number of Contracts</th> <th>Number of Contracts</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row["val"]), 2); $value = number_format(doubleval($row["val"]), 2);
$month_name = date('F', mktime(0, 0, 0, $row[1])); $month_name = date('F', mktime(0, 0, 0, $row[1]));
echo ("<tr><td><b><a href=\"?month=$row[1]-$row[0]\">$month_name {$row[0]}</a></b></td><td>\$$value</td><td>({$row['count']} contracts)</td></tr>"); echo ("<tr><td><b><a href=\"?month=$row[1]-$row[0]\">$month_name {$row[0]}</a></b></td><td>\$$value</td><td>({$row['count']} contracts)</td></tr>");
} }
echo "</table>"; echo "</table>";
} }
include_footer(); include_footer();
?> ?>
   
file:a/displayCategory.php -> file:b/displayCategory.php
<?php <?php
include_once ("./lib/common.inc.php"); include_once ("./lib/common.inc.php");
if ($_REQUEST['category']) { if ($_REQUEST['category']) {
include_header("Category"); include_header($_REQUEST['category']." goods and services");
echo "<center><h1>".$_REQUEST['category']."</h1></center>"; echo "<center><h1>".$_REQUEST['category']."</h1></center>";
$query = 'SELECT "CNID", description, value, "agencyName", category, "contractStart", "supplierName" $query = 'SELECT "CNID", description, value, "agencyName", category, "contractStart", "supplierName"
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
AND category = :category AND category = :category
ORDER BY value DESC'; ORDER BY value DESC';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":category", $_REQUEST['category']); $query->bindParam(":category", $_REQUEST['category']);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['value']) , 2); $value = number_format(doubleval($row['value']) , 2);
echo ("<tr> echo ("<tr>
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td>
<td><b>{$row['description']}</b></a></td> <td><b>{$row['description']}</b></a></td>
<td>\$$value</td><td>{$row['agencyName']}</td> <td>\$$value</td><td>{$row['agencyName']}</td>
<td>{$row['contractStart']}</td> <td>{$row['contractStart']}</td>
<td>{$row['supplierName']}</td> <td>{$row['supplierName']}</td>
</tr>"); </tr>");
} }
echo "</table>"; echo "</table>";
} else { } else {
/* /*
split by main categories split by main categories
*/ */
include_header("Categories"); include_header("Categories");
$query = 'SELECT sum(value), category $query = 'SELECT