<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
<modelVersion>4.0.0</modelVersion> | <modelVersion>4.0.0</modelVersion> |
<groupId>org.lambdacomplex.contractdashboard</groupId> | <groupId>org.lambdacomplex.contractdashboard</groupId> |
<artifactId>neo4jimporter</artifactId> | <artifactId>neo4jimporter</artifactId> |
<version>0.0.1-SNAPSHOT</version> | <version>0.0.1-SNAPSHOT</version> |
<dependencies> | <dependencies> |
<dependency> | <dependency> |
<groupId>org.neo4j</groupId> | <groupId>org.neo4j</groupId> |
<artifactId>neo4j</artifactId> | <artifactId>neo4j</artifactId> |
<version>1.5</version> | <version>1.6.1</version> |
</dependency> | </dependency> |
<dependency> | <dependency> |
<groupId>postgresql</groupId> | <groupId>postgresql</groupId> |
<artifactId>postgresql</artifactId> | <artifactId>postgresql</artifactId> |
<version>9.0-801.jdbc4</version> | <version>9.0-801.jdbc4</version> |
</dependency> | </dependency> |
</dependencies> | </dependencies> |
</project> | </project> |
import java.io.ObjectInputStream.GetField; | import java.io.ObjectInputStream.GetField; |
import java.math.BigInteger; | import java.math.BigInteger; |
import java.security.MessageDigest; | import java.security.MessageDigest; |
import java.security.NoSuchAlgorithmException; | import java.security.NoSuchAlgorithmException; |
import java.sql.Connection; | import java.sql.Connection; |
import java.sql.DriverManager; | import java.sql.DriverManager; |
import java.sql.ResultSet; | import java.sql.ResultSet; |
import java.sql.SQLException; | import java.sql.SQLException; |
import java.sql.SQLWarning; | import java.sql.SQLWarning; |
import java.sql.Statement; | import java.sql.Statement; |
import java.util.HashMap; | import java.util.HashMap; |
import java.util.Map; | import java.util.Map; |
import org.neo4j.graphdb.DynamicRelationshipType; | import org.neo4j.graphdb.DynamicRelationshipType; |
import org.neo4j.graphdb.GraphDatabaseService; | import org.neo4j.graphdb.GraphDatabaseService; |
import org.neo4j.graphdb.Node; | import org.neo4j.graphdb.Node; |
import org.neo4j.graphdb.index.BatchInserterIndex; | import org.neo4j.graphdb.index.BatchInserterIndex; |
import org.neo4j.graphdb.index.BatchInserterIndexProvider; | import org.neo4j.graphdb.index.BatchInserterIndexProvider; |
import org.neo4j.helpers.collection.MapUtil; | import org.neo4j.helpers.collection.MapUtil; |
import org.neo4j.kernel.impl.batchinsert.BatchInserter; | import org.neo4j.kernel.impl.batchinsert.BatchInserter; |
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; | import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; |
import org.neo4j.index.impl.lucene.*; | import org.neo4j.index.impl.lucene.*; |
public class Importer { | public class Importer { |
public static void main(String[] argv) { | public static void main(String[] argv) { |
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); | BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); |
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); | BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); |
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); | BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); |
labels.setCacheCapacity("Label", 100000); | labels.setCacheCapacity("Label", 100000); |
System.out.println("-------- PostgreSQL " | System.out.println("-------- PostgreSQL " |
+ "JDBC Connection Testing ------------"); | + "JDBC Connection Testing ------------"); |
try { | try { |
Class.forName("org.postgresql.Driver"); | Class.forName("org.postgresql.Driver"); |
} catch (ClassNotFoundException e) { | } catch (ClassNotFoundException e) { |
System.out.println("Where is your PostgreSQL JDBC Driver? " | System.out.println("Where is your PostgreSQL JDBC Driver? " |
+ "Include in your library path!"); | + "Include in your library path!"); |
e.printStackTrace(); | e.printStackTrace(); |
} | } |
System.out.println("PostgreSQL JDBC Driver Registered!"); | System.out.println("PostgreSQL JDBC Driver Registered!"); |
Connection conn = null; | Connection conn = null; |
try { | try { |
conn = DriverManager.getConnection( | conn = DriverManager.getConnection( |
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", | "jdbc:postgresql://127.0.0.1:5432/contractDashboard", |
"postgres", "snmc"); | "postgres", "snmc"); |
} catch (SQLException e) { | } catch (SQLException e) { |
System.out.println("Connection Failed! Check output console"); | System.out.println("Connection Failed! Check output console"); |
e.printStackTrace(); | e.printStackTrace(); |
} | } |
if (conn != null) { | if (conn != null) { |
System.out.println("You made it, take control your database now!"); | System.out.println("You made it, take control your database now!"); |
} else { | } else { |
System.out.println("Failed to make connection!"); | System.out.println("Failed to make connection!"); |
} | } |
try { | try { |
// Print all warnings | // Print all warnings |
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { | for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { |
System.out.println("SQL Warning:"); | System.out.println("SQL Warning:"); |
System.out.println("State : " + warn.getSQLState()); | System.out.println("State : " + warn.getSQLState()); |
System.out.println("Message: " + warn.getMessage()); | System.out.println("Message: " + warn.getMessage()); |
System.out.println("Error : " + warn.getErrorCode()); | System.out.println("Error : " + warn.getErrorCode()); |
} | } |
// Get a statement from the connection | // Get a statement from the connection |
Statement stmt = conn.createStatement(); | Statement stmt = conn.createStatement(); |
// Execute the query | // Execute the query |
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " | ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " |
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " | + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " |
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" | + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" |
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", " | + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", " |
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); | + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); |
String previousAgency = ""; | String previousAgency = ""; |
GraphDatabaseService gds = inserter.getGraphDbService(); | GraphDatabaseService gds = inserter.getGraphDbService(); |
HashMap<String,Long> supplierIDs = new HashMap<String,Long>(); | HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); |
HashMap<String,Long> agencyIDs = new HashMap<String,Long>(); | HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); |
// Loop through the result set | // Loop through the result set |
while (rs.next()) { | while (rs.next()) { |
long supplierID, agencyID; | long supplierID, agencyID; |
String supplierKey; | String supplierKey; |
if (agencyIDs.get(rs.getString("agencyName")) == null) { | if (agencyIDs.get(rs.getString("agencyName")) == null) { |
Node myNode = gds.createNode(); | Node myNode = gds.createNode(); |
myNode.setProperty("Label", rs.getString("agencyName")); | myNode.setProperty("Label", rs.getString("agencyName")); |
myNode.setProperty("type", "agency"); | myNode.setProperty("type", "agency"); |
agencyIDs.put(rs.getString("agencyName"), myNode.getId()); | agencyIDs.put(rs.getString("agencyName"), myNode.getId()); |
if (myNode.getId() %100 == 0) { | if (myNode.getId() % 100 == 0) { |
System.out.println("Agency "+myNode.getId()); | System.out.println("Agency " + myNode.getId()); |
} | } |
} | } |
agencyID = agencyIDs.get(rs.getString("agencyName")); | agencyID = agencyIDs.get(rs.getString("agencyName")); |
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { | if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { |
supplierKey = rs.getString("supplierABN"); | supplierKey = rs.getString("supplierABN"); |
} else { | } else { |
supplierKey = rs.getString("supplierName"); | supplierKey = rs.getString("supplierName"); |
} | } |
// inject some data | // inject some data |
if (supplierIDs.get(supplierKey) == null) { | if (supplierIDs.get(supplierKey) == null) { |
Node myNode = gds.createNode(); | Node myNode = gds.createNode(); |
myNode.setProperty("Label", rs.getString("supplierName")); | myNode.setProperty("Label", rs.getString("supplierName")); |
myNode.setProperty("type", "supplier"); | myNode.setProperty("type", "supplier"); |
supplierIDs.put(supplierKey, myNode.getId()); | supplierIDs.put(supplierKey, myNode.getId()); |
if (myNode.getId() %1000 == 0) { | if (myNode.getId() % 1000 == 0) { |
System.out.println("Supplier "+myNode.getId()); | System.out.println("Supplier " + myNode.getId()); |
} | } |
} | } |
supplierID = supplierIDs.get(supplierKey); | supplierID = supplierIDs.get(supplierKey); |
long rel = inserter.createRelationship(agencyID, supplierID, | long rel = inserter.createRelationship(agencyID, supplierID, |
DynamicRelationshipType.withName("KNOWS"), null); | DynamicRelationshipType.withName("KNOWS"), null); |
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); | inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); |
} | } |
// Close the result set, statement and the connection | // Close the result set, statement and the connection |
rs.close(); | rs.close(); |
stmt.close(); | stmt.close(); |
conn.close(); | conn.close(); |
} catch (SQLException se) { | } catch (SQLException se) { |
System.out.println("SQL Exception:"); | System.out.println("SQL Exception:"); |
// Loop through the SQL Exceptions | // Loop through the SQL Exceptions |
while (se != null) { | while (se != null) { |
System.out.println("State : " + se.getSQLState()); | System.out.println("State : " + se.getSQLState()); |
System.out.println("Message: " + se.getMessage()); | System.out.println("Message: " + se.getMessage()); |
System.out.println("Error : " + se.getErrorCode()); | System.out.println("Error : " + se.getErrorCode()); |
se = se.getNextException(); | se = se.getNextException(); |
} | } |
} | } |
//make the changes visible for reading, use this sparsely, requires IO! | //make the changes visible for reading, use this sparsely, requires IO! |
labels.flush(); | labels.flush(); |
// Make sure to shut down the index provider | // Make sure to shut down the index provider |
indexProvider.shutdown(); | indexProvider.shutdown(); |
inserter.shutdown(); | inserter.shutdown(); |
} | } |
} | } |
Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
<?php | <?php |
include_once("./lib/common.inc.php"); | include_once("./lib/common.inc.php"); |
include_header("Months and Years"); | include_header("Months and Years"); |
if ($_REQUEST['month']) { | if ($_REQUEST['month']) { |
echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>"; | echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>"; |
$monthParts = explode("-", $_REQUEST['month']); | $monthParts = explode("-", $_REQUEST['month']); |
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName" | $query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName" |
FROM contractnotice | FROM contractnotice |
WHERE "childCN" is null | WHERE "childCN" is null |
AND extract(year from "contractStart") = :year | AND extract(year from "contractStart") = :year |
AND extract(month from "contractStart") = :month | AND extract(month from "contractStart") = :month |
ORDER BY value DESC'; | ORDER BY value DESC'; |
$query = $conn->prepare($query); | $query = $conn->prepare($query); |
$query->bindParam(":month", $monthParts[0]); | $query->bindParam(":month", $monthParts[0]); |
$query->bindParam(":year", $monthParts[1]); | $query->bindParam(":year", $monthParts[1]); |
$query->execute(); | $query->execute(); |
databaseError($conn->errorInfo()); | databaseError($conn->errorInfo()); |
MethodCountGraph($supplier); | //MethodCountGraph($supplier); |
CnCGraph($supplier); | //CnCGraph($supplier); |
echo "<table> <thead> | echo "<table> <thead> |
<tr> | <tr> |
<th>Contract Notice Number</th> | <th>Contract Notice Number</th> |
<th>Contract Description</th> | <th>Contract Description</th> |
<th>Total Contract Value</th> | <th>Total Contract Value</th> |
<th>Agency</th> | <th>Agency</th> |
<th>Contract Start Date</th> | <th>Contract Start Date</th> |
<th>Supplier</th> | <th>Supplier</th> |
</tr> | </tr> |
</thead>"; | </thead>"; |
foreach ($query->fetchAll() as $row) { | foreach ($query->fetchAll() as $row) { |
setlocale(LC_MONETARY, 'en_US'); | setlocale(LC_MONETARY, 'en_US'); |
$value = number_format(doubleval($row['value']), 2); | $value = number_format(doubleval($row['value']), 2); |
echo ("<tr> | echo ("<tr> |
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> | <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> |
<td><b>{$row['description']}</b></a></td> | <td><b>{$row['description']}</b></a></td> |
<td>\$$value</td><td>{$row['agencyName']}</td> | <td>\$$value</td><td>{$row['agencyName']}</td> |
<td>{$row['contractStart']}</td> | <td>{$row['contractStart']}</td> |
<td>{$row['supplierName']}</td> | <td>{$row['supplierName']}</td> |
</tr>"); | </tr>"); |
} | } |
echo "</table>"; | echo "</table>"; |
} else { | } else { |
/* | /* |
split by year | split by year |
todo: | todo: |
Year/Month drilldown - largest contracts, agencies, suppliers | Year/Month drilldown - largest contracts, agencies, suppliers |
count per month | count per month |
big picture graphs? */ | big picture graphs? */ |
echo '<img src="graphs/displayContractStartingGraph.php">'; | echo '<img src="graphs/displayContractStartingGraph.php">'; |
echo '<img src="graphs/displayContractPublishedGraph.php">'; | echo '<img src="graphs/displayContractPublishedGraph.php">'; |
$query = 'SELECT extract(year from "contractStart"), extract(month from "contractStart"), | $query = 'SELECT extract(year from "contractStart"), extract(month from "contractStart"), |
SUM(value) as val, count(1) as count FROM contractnotice WHERE "childCN" is null GROUP BY extract(year from "contractStart"), extract(month from "contractStart") ORDER BY extract(year from "contractStart"), extract(month from "contractStart") '; | SUM(value) as val, count(1) as count FROM contractnotice WHERE "childCN" is null GROUP BY extract(year from "contractStart"), extract(month from "contractStart") ORDER BY extract(year from "contractStart"), extract(month from "contractStart") '; |
$query = $conn->prepare($query); | $query = $conn->prepare($query); |
$query->execute(); | $query->execute(); |
databaseError($conn->errorInfo()); | databaseError($conn->errorInfo()); |
echo "<table> <thead> | echo "<table> <thead> |
<tr> | <tr> |
<th>Month/Year</th> | <th>Month/Year</th> |
<th>Total Contracts Value</th> | <th>Total Contracts Value</th> |
<th>Number of Contracts</th> | <th>Number of Contracts</th> |
</tr> | </tr> |
</thead>"; | </thead>"; |
foreach ($query->fetchAll() as $row) { | foreach ($query->fetchAll() as $row) { |
setlocale(LC_MONETARY, 'en_US'); | setlocale(LC_MONETARY, 'en_US'); |
$value = number_format(doubleval($row["val"]), 2); | $value = number_format(doubleval($row["val"]), 2); |
$month_name = date('F', mktime(0, 0, 0, $row[1])); | $month_name = date('F', mktime(0, 0, 0, $row[1])); |
echo ("<tr><td><b><a href=\"?month=$row[1]-$row[0]\">$month_name {$row[0]}</a></b></td><td>\$$value</td><td>({$row['count']} contracts)</td></tr>"); | echo ("<tr><td><b><a href=\"?month=$row[1]-$row[0]\">$month_name {$row[0]}</a></b></td><td>\$$value</td><td>({$row['count']} contracts)</td></tr>"); |
} | } |
echo "</table>"; | echo "</table>"; |
} | } |
include_footer(); | include_footer(); |
?> | ?> |
<?php | <?php |
include_once("./lib/common.inc.php"); | include_once("./lib/common.inc.php"); |
setlocale(LC_CTYPE, 'C'); | setlocale(LC_CTYPE, 'C'); |
// source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 | // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951 |
$unspsc = Array(); | $unspsc = Array(); |
$unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';'); | $unspscresult = $conn->prepare('select * from "UNSPSCcategories" where "UNSPSC"::text like \'%00000\';'); |
$unspscresult->execute(); | $unspscresult->execute(); |
foreach ($unspscresult->fetchAll() as $row) { | foreach ($unspscresult->fetchAll() as $row) { |
$unspsc[$row['UNSPSC']] = $row['Title']; | $unspsc[$row['UNSPSC']] = $row['Title']; |
} | } |
/*SELECT count(*), extract("week" from "publishDate") as week,extract("year" from "publishDate") as year | |
FROM contractnotice | |
where "childCN" is null group by week, year order by year, week*/ | |
$query = $conn->prepare(' | $query = $conn->prepare(' |
SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN", | SELECT "CNID",contractnotice."agencyName",agency_nametoabn.abn as "agencyABN", |
EXTRACT(EPOCH FROM "publishDate") as "publishDate", | EXTRACT(EPOCH FROM "publishDate") as "publishDate", |
EXTRACT(EPOCH FROM "contractStart") as "contractStart", | EXTRACT(EPOCH FROM "contractStart") as "contractStart", |
EXTRACT(EPOCH FROM "contractEnd") as "contractEnd", | EXTRACT(EPOCH FROM "contractEnd") as "contractEnd", |
value,description,category, | value,description,category, |
"supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, | "supplierName",(case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID, |
(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL | (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL |
FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" | FROM contractnotice join agency_nametoabn on contractnotice."agencyName"=agency_nametoabn."agencyName" |
where "childCN" is null' | where "childCN" is null' |
, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); | , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT)); |
$query->execute(); | $query->execute(); |
$errors = $conn->errorInfo(); | $errors = $conn->errorInfo(); |
if ($errors[2] != "") { | if ($errors[2] != "") { |
die("Export terminated, db error" . print_r($errors, true)); | die("Export terminated, db error" . print_r($errors, true)); |