projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Upgrade neo4j batchimporter to 1.5
Upgrade neo4j batchimporter to 1.5

file:b/admin/neo4jimporter/.classpath (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <classpath>
  <classpathentry kind="src" output="target/classes" path="src/main/java"/>
  <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
  <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
  <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
  <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
  <classpathentry kind="output" path="target/classes"/>
  </classpath>
 
file:b/admin/neo4jimporter/.project (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <projectDescription>
  <name>neo4jimporter</name>
  <comment></comment>
  <projects>
  </projects>
  <buildSpec>
  <buildCommand>
  <name>org.eclipse.jdt.core.javabuilder</name>
  <arguments>
  </arguments>
  </buildCommand>
  <buildCommand>
  <name>org.eclipse.m2e.core.maven2Builder</name>
  <arguments>
  </arguments>
  </buildCommand>
  </buildSpec>
  <natures>
  <nature>org.eclipse.jdt.core.javanature</nature>
  <nature>org.eclipse.m2e.core.maven2Nature</nature>
  </natures>
  </projectDescription>
 
file:b/admin/neo4jimporter/.settings/org.eclipse.jdt.core.prefs (new)
  #Sun Aug 07 18:15:32 EST 2011
  eclipse.preferences.version=1
  org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
  org.eclipse.jdt.core.compiler.compliance=1.5
  org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
  org.eclipse.jdt.core.compiler.source=1.5
 
file:b/admin/neo4jimporter/.settings/org.eclipse.m2e.core.prefs (new)
  #Sun Aug 07 18:14:30 EST 2011
  activeProfiles=
  eclipse.preferences.version=1
  resolveWorkspaceProjects=true
  version=1
 
file:b/admin/neo4jimporter/nbactions.xml (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <actions>
  <action>
  <actionName>run</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>debug</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <jpda.listen>true</jpda.listen>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>profile</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
  </goals>
  <properties>
  <exec.args>${profiler.args} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
  <profiler.action>profile</profiler.action>
  <exec.executable>${profiler.java}</exec.executable>
  </properties>
  </action>
  <action>
  <actionName>run</actionName>
  <goals>
  <goal>process-classes</goal>
  <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
  </goals>
  <properties>
  <exec.classpathScope>runtime</exec.classpathScope>
  <exec.args>-classpath %classpath Importer</exec.args>
  <exec.executable>java</exec.executable>
  </properties>
  </action>
  </actions>
 
file:b/admin/neo4jimporter/pom.xml (new)
  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>org.lambdacomplex.contractdashboard</groupId>
  <artifactId>neo4jimporter</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <dependencies>
  <dependency>
  <groupId>org.neo4j</groupId>
  <artifactId>neo4j</artifactId>
  <version>1.5</version>
  </dependency>
  <dependency>
  <groupId>postgresql</groupId>
  <artifactId>postgresql</artifactId>
  <version>9.0-801.jdbc4</version>
  </dependency>
  </dependencies>
  </project>
file:b/admin/neo4jimporter/src/main/java/Importer.java (new)
 
  import java.io.ObjectInputStream.GetField;
  import java.math.BigInteger;
  import java.security.MessageDigest;
  import java.security.NoSuchAlgorithmException;
  import java.sql.Connection;
  import java.sql.DriverManager;
  import java.sql.ResultSet;
  import java.sql.SQLException;
  import java.sql.SQLWarning;
  import java.sql.Statement;
  import java.util.HashMap;
  import java.util.Map;
 
  import org.neo4j.graphdb.DynamicRelationshipType;
  import org.neo4j.graphdb.GraphDatabaseService;
  import org.neo4j.graphdb.Node;
  import org.neo4j.graphdb.index.BatchInserterIndex;
  import org.neo4j.graphdb.index.BatchInserterIndexProvider;
  import org.neo4j.helpers.collection.MapUtil;
  import org.neo4j.kernel.impl.batchinsert.BatchInserter;
  import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
  import org.neo4j.index.impl.lucene.*;
 
  public class Importer {
 
  public static void main(String[] argv) {
  BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
  BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
  BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
  labels.setCacheCapacity("Label", 100000);
 
 
 
  System.out.println("-------- PostgreSQL "
  + "JDBC Connection Testing ------------");
 
  try {
 
  Class.forName("org.postgresql.Driver");
 
  } catch (ClassNotFoundException e) {
 
  System.out.println("Where is your PostgreSQL JDBC Driver? "
  + "Include in your library path!");
  e.printStackTrace();
 
  }
 
  System.out.println("PostgreSQL JDBC Driver Registered!");
 
  Connection conn = null;
 
  try {
 
  conn = DriverManager.getConnection(
  "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
  "postgres", "snmc");
 
  } catch (SQLException e) {
 
  System.out.println("Connection Failed! Check output console");
  e.printStackTrace();
 
  }
 
  if (conn != null) {
  System.out.println("You made it, take control your database now!");
  } else {
  System.out.println("Failed to make connection!");
  }
  try {
  // Print all warnings
  for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
  System.out.println("SQL Warning:");
  System.out.println("State : " + warn.getSQLState());
  System.out.println("Message: " + warn.getMessage());
  System.out.println("Error : " + warn.getErrorCode());
  }
 
  // Get a statement from the connection
  Statement stmt = conn.createStatement();
 
  // Execute the query
  ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
  + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
  + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
  + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
  + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
  String previousAgency = "";
  GraphDatabaseService gds = inserter.getGraphDbService();
  HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
  HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
 
  // Loop through the result set
  while (rs.next()) {
  long supplierID, agencyID;
  String supplierKey;
  if (agencyIDs.get(rs.getString("agencyName")) == null) {
  Node myNode = gds.createNode();
  myNode.setProperty("Label", rs.getString("agencyName"));
  myNode.setProperty("type", "agency");
  agencyIDs.put(rs.getString("agencyName"), myNode.getId());
  if (myNode.getId() %100 == 0) {
  System.out.println("Agency "+myNode.getId());
  }
  }
  agencyID = agencyIDs.get(rs.getString("agencyName"));
 
 
  if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
  supplierKey = rs.getString("supplierABN");
  } else {
  supplierKey = rs.getString("supplierName");
  }
  // inject some data
  if (supplierIDs.get(supplierKey) == null) {
  Node myNode = gds.createNode();
  myNode.setProperty("Label", rs.getString("supplierName"));
  myNode.setProperty("type", "supplier");
  supplierIDs.put(supplierKey, myNode.getId());
  if (myNode.getId() %1000 == 0) {
  System.out.println("Supplier "+myNode.getId());
  }
  }
  supplierID = supplierIDs.get(supplierKey);
 
 
  long rel = inserter.createRelationship(agencyID, supplierID,
  DynamicRelationshipType.withName("KNOWS"), null);
  inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
 
  }
  // Close the result set, statement and the connection
  rs.close();
  stmt.close();
  conn.close();
  } catch (SQLException se) {
  System.out.println("SQL Exception:");
 
  // Loop through the SQL Exceptions
  while (se != null) {
  System.out.println("State : " + se.getSQLState());
  System.out.println("Message: " + se.getMessage());
  System.out.println("Error : " + se.getErrorCode());
 
  se = se.getNextException();
  }
  }
  //make the changes visible for reading, use this sparsely, requires IO!
  labels.flush();
 
  // Make sure to shut down the index provider
  indexProvider.shutdown();
  inserter.shutdown();
  }
 
 
  }
file:b/admin/neo4jimporter/src/main/java/JDBCExample.java (new)
  import java.sql.DriverManager;
  import java.sql.Connection;
  import java.sql.SQLException;
 
  public class JDBCExample {
 
  public static void main(String[] argv) {
 
  System.out.println("-------- PostgreSQL "
  + "JDBC Connection Testing ------------");
 
  try {
 
  Class.forName("org.postgresql.Driver");
 
  } catch (ClassNotFoundException e) {
 
  System.out.println("Where is your PostgreSQL JDBC Driver? "
  + "Include in your library path!");
  e.printStackTrace();
  return;
 
  }
 
  System.out.println("PostgreSQL JDBC Driver Registered!");
 
  Connection connection = null;
 
  try {
 
  connection = DriverManager.getConnection(
  "jdbc:postgresql://127.0.0.1:5432/contractDashboard", "postgres",
  "snmc");
 
  } catch (SQLException e) {
 
  System.out.println("Connection Failed! Check output console");
  e.printStackTrace();
  return;
 
  }
 
  if (connection != null) {
  System.out.println("You made it, take control your database now!");
  } else {
  System.out.println("Failed to make connection!");
  }
  }
 
  }
file:b/admin/neo4jimporter/target/classes/Importer.class (new) 
 Binary files /dev/null and b/admin/neo4jimporter/target/classes/Importer.class differ
file:b/admin/neo4jimporter/target/classes/JDBCExample.class (new) 
 Binary files /dev/null and b/admin/neo4jimporter/target/classes/JDBCExample.class differ
file:a/admin/partialdata/import.php -> file:b/admin/partialdata/import.php
<?php <?php
include_once ("../../lib/common.inc.php"); include_once ("../../lib/common.inc.php");
function processFile($fpath, $tablename) function processFile($fpath, $tablename)
{ {
global $conn; global $conn;
echo " ============== $fpath ============== <br>"; echo " ============== $fpath ============== <br>";
flush(); flush();
$row = 1; $row = 1;
$success = 0; $success = 0;
$dupes = 0; $dupes = 0;
$handle = fopen($fpath, "r"); $handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix //"t" mode string translates windows line breaks to unix
$datamapping0507 = array( $datamapping0507 = array(
"Agency" => "agencyName", "Agency" => "agencyName",
"CN ID" => "CNID", "CN ID" => "CNID",
"Publish Date" => "publishDate", "Publish Date" => "publishDate",
"Contract Start Date" => "contractStart", "Contract Start Date" => "contractStart",
"Contract End Date" => "contractEnd", "Contract End Date" => "contractEnd",
"Value (AUD)" => "value", "Value (AUD)" => "value",
"Title" => "description", "Title" => "description",
"Category" => "category", "Category" => "category",
"ATM ID" => "atmID", "ATM ID" => "atmID",