projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Upgrade neo4j batchimporter to 1.5
Upgrade neo4j batchimporter to 1.5

file:b/admin/neo4jimporter/nbactions.xml (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <actions>
  <action>
  <actionName>run</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>debug</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <jpda.listen>true</jpda.listen>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>profile</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
  </goals>
  <properties>
  <exec.args>${profiler.args} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <profiler.action>profile</profiler.action>
  <exec.executable>${profiler.java}</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>run</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-classpath %classpath Importer</exec.args>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  </actions>
 
file:a/admin/neo4jimporter/pom.xml -> file:b/admin/neo4jimporter/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.lambdacomplex.contractdashboard</groupId> <groupId>org.lambdacomplex.contractdashboard</groupId>
<artifactId>neo4jimporter</artifactId> <artifactId>neo4jimporter</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j-kernel</artifactId> <artifactId>neo4j</artifactId>
<version>1.4</version> <version>1.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>postgresql</groupId> <groupId>postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<version>9.0-801.jdbc4</version> <version>9.0-801.jdbc4</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>
file:a/admin/neo4jimporter/src/main/java/Importer.java -> file:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.io.ObjectInputStream.GetField; import java.io.ObjectInputStream.GetField;
import java.math.BigInteger; import java.math.BigInteger;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
   
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
  import org.neo4j.graphdb.GraphDatabaseService;
  import org.neo4j.graphdb.Node;
  import org.neo4j.graphdb.index.BatchInserterIndex;
  import org.neo4j.graphdb.index.BatchInserterIndexProvider;
  import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
  import org.neo4j.index.impl.lucene.*;
   
public class Importer { public class Importer {
   
public static void main(String[] argv) { public static void main(String[] argv) {
  BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
Map<String,String> props = new HashMap<String, String>(); BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes> BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes> labels.setCacheCapacity("Label", 100000);
// create the batch inserter  
BatchInserter inserter = new  
BatchInserterImpl("neo4j-db/",props  
);  
   
   
   
   
   
System.out.println("-------- PostgreSQL "  
+ "JDBC Connection Testing ------------");  
   
try { System.out.println("-------- PostgreSQL "
  + "JDBC Connection Testing ------------");
   
Class.forName("org.postgresql.Driver"); try {
   
} catch (ClassNotFoundException e) { Class.forName("org.postgresql.Driver");
   
System.out.println("Where is your PostgreSQL JDBC Driver? " } catch (ClassNotFoundException e) {
+ "Include in your library path!");  
e.printStackTrace();  
   
} System.out.println("Where is your PostgreSQL JDBC Driver? "
  + "Include in your library path!");
  e.printStackTrace();
   
System.out.println("PostgreSQL JDBC Driver Registered!"); }
   
Connection conn = null; System.out.println("PostgreSQL JDBC Driver Registered!");
   
try { Connection conn = null;
   
conn = DriverManager.getConnection( try {
"jdbc:postgresql://127.0.0.1:5432/contractDashboard",  
"postgres", "snmc");  
   
} catch (SQLException e) { conn = DriverManager.getConnection(
  "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
  "postgres", "snmc");
   
System.out.println("Connection Failed! Check output console"); } catch (SQLException e) {
e.printStackTrace();  
   
} System.out.println("Connection Failed! Check output console");
  e.printStackTrace();
   
if (conn != null) { }
System.out.println("You made it, take control your database now!");  
} else {  
System.out.println("Failed to make connection!");  
}  
try {  
// Print all warnings  
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn  
.getNextWarning()) {  
System.out.println("SQL Warning:");  
System.out.println("State : " + warn.getSQLState());  
System.out.println("Message: " + warn.getMessage());  
System.out.println("Error : " + warn.getErrorCode());  
}  
   
// Get a statement from the connection if (conn != null) {
Statement stmt = conn.createStatement(); System.out.println("You made it, take control your database now!");
  } else {
  System.out.println("Failed to make connection!");
  }
  try {
  // Print all warnings
  for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
  System.out.println("SQL Warning:");
  System.out.println("State : " + warn.getSQLState());
  System.out.println("Message: " + warn.getMessage());
  System.out.println("Error : " + warn.getErrorCode());
  }
   
// Execute the query // Get a statement from the connection
ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;"); Statement stmt = conn.createStatement();
String previousAgency = "";  
   
// Loop through the result set // Execute the query
while (rs.next()) { ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
long supplier,agency; + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
agency = doHash(rs.getString("agencyName")); + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
if (rs.getString("agencyName") != previousAgency) { + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
if (!inserter.nodeExists(agency)) { + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
Map<String, Object> properties = new HashMap<String, Object>(); String previousAgency = "";
properties.put("Label", rs.getString("agencyName")); GraphDatabaseService gds = inserter.getGraphDbService();
inserter.createNode(agency, properties); HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
} HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
}  
if (rs.getString("supplierABN") != "0") { // Loop through the result set
supplier = doHash(rs.getString("supplierABN")); while (rs.next()) {
} else { long supplierID, agencyID;
supplier = doHash(rs.getString("supplierName")); String supplierKey;
} if (agencyIDs.get(rs.getString("agencyName")) == null) {
// inject some data Node myNode = gds.createNode();
if (!inserter.nodeExists(supplier)) { myNode.setProperty("Label", rs.getString("agencyName"));
Map<String, Object> properties = new HashMap<String, Object>(); myNode.setProperty("type", "agency");
  agencyIDs.put(rs.getString("agencyName"), myNode.getId());
properties.put("Label", rs.getString("supplierName")); if (myNode.getId() %100 == 0) {
inserter.createNode(supplier, properties); System.out.println("Agency "+myNode.getId());
} }
  }
inserter.createRelationship(agency, supplier, agencyID = agencyIDs.get(rs.getString("agencyName"));
DynamicRelationshipType.withName("KNOWS"), null);  
   
} if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
// Close the result set, statement and the connection supplierKey = rs.getString("supplierABN");
rs.close(); } else {
stmt.close(); supplierKey = rs.getString("supplierName");
conn.close(); }
} catch (SQLException se) { // inject some data
System.out.println("SQL Exception:"); if (supplierIDs.get(supplierKey) == null) {
  Node myNode = gds.createNode();
// Loop through the SQL Exceptions myNode.setProperty("Label", rs.getString("supplierName"));
while (se != null) { myNode.setProperty("type", "supplier");
System.out.println("State : " + se.getSQLState()); supplierIDs.put(supplierKey, myNode.getId());
System.out.println("Message: " + se.getMessage()); if (myNode.getId() %1000 == 0) {
System.out.println("Error : " + se.getErrorCode()); System.out.println("Supplier "+myNode.getId());
  }
se = se.getNextException(); }
} supplierID = supplierIDs.get(supplierKey);
} catch (Exception e) {  
System.out.println(e);  
}  
// shutdown, makes sure all changes are written to disk  
inserter.shutdown();  
}  
   
static long doHash(String input) {  
MessageDigest m;  
try {  
m = MessageDigest.getInstance("MD5");  
m.reset();  
m.update(input.getBytes());  
byte[] digest = m.digest();  
return new BigInteger(1, digest).longValue();  
} catch (NoSuchAlgorithmException e) {  
// TODO Auto-generated catch block  
e.printStackTrace();  
}  
return 0;  
   
}  
   
   
  long rel = inserter.createRelationship(agencyID, supplierID,
  DynamicRelationshipType.withName("KNOWS"), null);
  inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
   
  }
  // Close the result set, statement and the connection
  rs.close();
  stmt.close();
  conn.close();
  } catch (SQLException se) {
  System.out.println("SQL Exception:");
   
  // Loop through the SQL Exceptions
  while (se != null) {
  System.out.println("State : " + se.getSQLState());
  System.out.println("Message: " + se.getMessage());
  System.out.println("Error : " + se.getErrorCode());
   
  se = se.getNextException();
  }
  }
  //make the changes visible for reading, use this sparsely, requires IO!
  labels.flush();
   
  // Make sure to shut down the index provider
  indexProvider.shutdown();
  inserter.shutdown();
  }
   
   
} }
file:a/admin/neo4jimporter/target/classes/Importer.class -> file:b/admin/neo4jimporter/target/classes/Importer.class 
 Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
Unnamed repository; edit this file to name it for gitweb.
GitPHP by Chris Han