Fix more graphs
[contractdashboard.git] / admin / neo4jimporter / src / main / java / Importer.java
blob:a/admin/neo4jimporter/src/main/java/Importer.java -> blob:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.io.ObjectInputStream.GetField; import java.sql.Connection;
import java.math.BigInteger; import java.sql.DriverManager;
import java.security.MessageDigest; import java.sql.ResultSet;
import java.security.NoSuchAlgorithmException; import java.sql.SQLException;
import java.sql.Connection; import java.sql.SQLWarning;
import java.sql.DriverManager; import java.sql.Statement;
import java.sql.ResultSet; import java.util.HashMap;
import java.sql.SQLException;  
import java.sql.SQLWarning; import org.neo4j.graphdb.DynamicRelationshipType;
import java.sql.Statement; import org.neo4j.graphdb.GraphDatabaseService;
import java.util.HashMap; import org.neo4j.graphdb.Node;
import java.util.Map; import org.neo4j.graphdb.index.BatchInserterIndex;
  import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
import org.neo4j.graphdb.Node; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
import org.neo4j.graphdb.index.BatchInserterIndexProvider;  
import org.neo4j.helpers.collection.MapUtil; public class Importer {
import org.neo4j.kernel.impl.batchinsert.BatchInserter;  
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; public static void main(String[] argv) {
import org.neo4j.index.impl.lucene.*; BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
  BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
public class Importer { BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
  labels.setCacheCapacity("Label", 100000);
public static void main(String[] argv) {  
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");  
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);  
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); System.out.println("-------- PostgreSQL "
labels.setCacheCapacity("Label", 100000); + "JDBC Connection Testing ------------");
   
  try {
   
System.out.println("-------- PostgreSQL " Class.forName("org.postgresql.Driver");
+ "JDBC Connection Testing ------------");  
  } catch (ClassNotFoundException e) {
try {  
  System.out.println("Where is your PostgreSQL JDBC Driver? "
Class.forName("org.postgresql.Driver"); + "Include in your library path!");
  e.printStackTrace();
} catch (ClassNotFoundException e) {  
  }
System.out.println("Where is your PostgreSQL JDBC Driver? "  
+ "Include in your library path!"); System.out.println("PostgreSQL JDBC Driver Registered!");
e.printStackTrace();  
  Connection conn = null;
}  
  try {
System.out.println("PostgreSQL JDBC Driver Registered!");  
  conn = DriverManager.getConnection(
Connection conn = null; "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
  "postgres", "snmc");
try {  
  } catch (SQLException e) {
conn = DriverManager.getConnection(  
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", System.out.println("Connection Failed! Check output console");
"postgres", "snmc"); e.printStackTrace();
   
} catch (SQLException e) { }
   
System.out.println("Connection Failed! Check output console"); if (conn != null) {
e.printStackTrace(); System.out.println("You made it, take control your database now!");
  } else {
} System.out.println("Failed to make connection!");
  }
if (conn != null) { try {
System.out.println("You made it, take control your database now!"); // Print all warnings
} else { for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
System.out.println("Failed to make connection!"); System.out.println("SQL Warning:");
} System.out.println("State : " + warn.getSQLState());
try { System.out.println("Message: " + warn.getMessage());
// Print all warnings System.out.println("Error : " + warn.getErrorCode());
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { }
System.out.println("SQL Warning:");  
System.out.println("State : " + warn.getSQLState()); // Get a statement from the connection
System.out.println("Message: " + warn.getMessage()); Statement stmt = conn.createStatement();
System.out.println("Error : " + warn.getErrorCode());  
} // Execute the query
  ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
// Get a statement from the connection + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
Statement stmt = conn.createStatement(); + "FROM public.contractnotice GROUP BY contractnotice.\"agencyName\", "
  + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
// Execute the query String previousAgency = "";
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " GraphDatabaseService gds = inserter.getGraphDbService();
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "  
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); // Loop through the result set
String previousAgency = ""; while (rs.next()) {
GraphDatabaseService gds = inserter.getGraphDbService(); long supplierID, agencyID;
HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); String supplierKey;
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); if (agencyIDs.get(rs.getString("agencyName")) == null) {
  Node myNode = gds.createNode();
// Loop through the result set myNode.setProperty("Label", rs.getString("agencyName"));
while (rs.next()) { myNode.setProperty("type", "agency");
long supplierID, agencyID; agencyIDs.put(rs.getString("agencyName"), myNode.getId());
String supplierKey; if (myNode.getId() % 100 == 0) {
if (agencyIDs.get(rs.getString("agencyName")) == null) { System.out.println("Agency " + myNode.getId());
Node myNode = gds.createNode(); }
myNode.setProperty("Label", rs.getString("agencyName")); }
myNode.setProperty("type", "agency"); agencyID = agencyIDs.get(rs.getString("agencyName"));
agencyIDs.put(rs.getString("agencyName"), myNode.getId());  
if (myNode.getId() % 100 == 0) {  
System.out.println("Agency " + myNode.getId()); if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
} supplierKey = rs.getString("supplierABN");
} } else {
agencyID = agencyIDs.get(rs.getString("agencyName")); supplierKey = rs.getString("supplierName");
  }
  // inject some data
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { if (supplierIDs.get(supplierKey) == null) {
supplierKey = rs.getString("supplierABN"); Node myNode = gds.createNode();
} else { myNode.setProperty("Label", rs.getString("supplierName"));
supplierKey = rs.getString("supplierName"); myNode.setProperty("type", "supplier");
} supplierIDs.put(supplierKey, myNode.getId());
// inject some data if (myNode.getId() % 1000 == 0) {
if (supplierIDs.get(supplierKey) == null) { System.out.println("Supplier " + myNode.getId());
Node myNode = gds.createNode(); }
myNode.setProperty("Label", rs.getString("supplierName")); }
myNode.setProperty("type", "supplier"); supplierID = supplierIDs.get(supplierKey);
supplierIDs.put(supplierKey, myNode.getId());  
if (myNode.getId() % 1000 == 0) {  
System.out.println("Supplier " + myNode.getId()); long rel = inserter.createRelationship(agencyID, supplierID,
} DynamicRelationshipType.withName("KNOWS"), null);
} inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
supplierID = supplierIDs.get(supplierKey);  
  }
  // Close the result set, statement and the connection
long rel = inserter.createRelationship(agencyID, supplierID, rs.close();
DynamicRelationshipType.withName("KNOWS"), null); stmt.close();
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); conn.close();
  } catch (SQLException se) {
} System.out.println("SQL Exception:");
// Close the result set, statement and the connection  
rs.close(); // Loop through the SQL Exceptions
stmt.close(); while (se != null) {
conn.close(); System.out.println("State : " + se.getSQLState());
} catch (SQLException se) { System.out.println("Message: " + se.getMessage());
System.out.println("SQL Exception:"); System.out.println("Error : " + se.getErrorCode());
   
// Loop through the SQL Exceptions se = se.getNextException();
while (se != null) { }
System.out.println("State : " + se.getSQLState()); }
System.out.println("Message: " + se.getMessage()); //make the changes visible for reading, use this sparsely, requires IO!
System.out.println("Error : " + se.getErrorCode()); labels.flush();
   
se = se.getNextException(); // Make sure to shut down the index provider
} indexProvider.shutdown();
} inserter.shutdown();
//make the changes visible for reading, use this sparsely, requires IO! }
labels.flush();  
   
// Make sure to shut down the index provider  
indexProvider.shutdown();  
inserter.shutdown();  
}  
} }