<?xml version="1.0" encoding="UTF-8"?> | |
<classpath> | |
<classpathentry kind="src" output="target/classes" path="src/main/java"/> | |
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/> | |
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/> | |
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/> | |
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/> | |
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/> | |
<classpathentry kind="output" path="target/classes"/> | |
</classpath> | |
<?xml version="1.0" encoding="UTF-8"?> | |
<projectDescription> | |
<name>neo4jimporter</name> | |
<comment></comment> | |
<projects> | |
</projects> | |
<buildSpec> | |
<buildCommand> | |
<name>org.eclipse.jdt.core.javabuilder</name> | |
<arguments> | |
</arguments> | |
</buildCommand> | |
<buildCommand> | |
<name>org.eclipse.m2e.core.maven2Builder</name> | |
<arguments> | |
</arguments> | |
</buildCommand> | |
</buildSpec> | |
<natures> | |
<nature>org.eclipse.jdt.core.javanature</nature> | |
<nature>org.eclipse.m2e.core.maven2Nature</nature> | |
</natures> | |
</projectDescription> | |
#Sun Aug 07 18:15:32 EST 2011 | |
eclipse.preferences.version=1 | |
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 | |
org.eclipse.jdt.core.compiler.compliance=1.5 | |
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning | |
org.eclipse.jdt.core.compiler.source=1.5 | |
#Sun Aug 07 18:14:30 EST 2011 | |
activeProfiles= | |
eclipse.preferences.version=1 | |
resolveWorkspaceProjects=true | |
version=1 | |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |
<modelVersion>4.0.0</modelVersion> | |
<groupId>org.lambdacomplex.contractdashboard</groupId> | |
<artifactId>neo4jimporter</artifactId> | |
<version>0.0.1-SNAPSHOT</version> | |
<dependencies> | |
<dependency> | |
<groupId>org.neo4j</groupId> | |
<artifactId>neo4j-kernel</artifactId> | |
<version>1.4</version> | |
</dependency> | |
<dependency> | |
<groupId>postgresql</groupId> | |
<artifactId>postgresql</artifactId> | |
<version>9.0-801.jdbc4</version> | |
</dependency> | |
</dependencies> | |
</project> |
import java.io.ObjectInputStream.GetField; | |
import java.math.BigInteger; | |
import java.security.MessageDigest; | |
import java.security.NoSuchAlgorithmException; | |
import java.sql.Connection; | |
import java.sql.DriverManager; | |
import java.sql.ResultSet; | |
import java.sql.SQLException; | |
import java.sql.SQLWarning; | |
import java.sql.Statement; | |
import java.util.HashMap; | |
import java.util.Map; | |
import org.neo4j.graphdb.DynamicRelationshipType; | |
import org.neo4j.kernel.impl.batchinsert.BatchInserter; | |
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; | |
public class Importer { | |
public static void main(String[] argv) { | |
Map<String,String> props = new HashMap<String, String>(); | |
props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes> | |
props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes> | |
// create the batch inserter | |
BatchInserter inserter = new | |
BatchInserterImpl("neo4j-db/",props | |
); | |
System.out.println("-------- PostgreSQL " | |
+ "JDBC Connection Testing ------------"); | |
try { | |
Class.forName("org.postgresql.Driver"); | |
} catch (ClassNotFoundException e) { | |
System.out.println("Where is your PostgreSQL JDBC Driver? " | |
+ "Include in your library path!"); | |
e.printStackTrace(); | |
} | |
System.out.println("PostgreSQL JDBC Driver Registered!"); | |
Connection conn = null; | |
try { | |
conn = DriverManager.getConnection( | |
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", | |
"postgres", "snmc"); | |
} catch (SQLException e) { | |
System.out.println("Connection Failed! Check output console"); | |
e.printStackTrace(); | |
} | |
if (conn != null) { | |
System.out.println("You made it, take control your database now!"); | |
} else { | |
System.out.println("Failed to make connection!"); | |
} | |
try { | |
// Print all warnings | |
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn | |
.getNextWarning()) { | |
System.out.println("SQL Warning:"); | |
System.out.println("State : " + warn.getSQLState()); | |
System.out.println("Message: " + warn.getMessage()); | |
System.out.println("Error : " + warn.getErrorCode()); | |
} | |
// Get a statement from the connection | |
Statement stmt = conn.createStatement(); | |
// Execute the query | |
ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;"); | |
String previousAgency = ""; | |
// Loop through the result set | |
while (rs.next()) { | |
long supplier,agency; | |
agency = doHash(rs.getString("agencyName")); | |
if (rs.getString("agencyName") != previousAgency) { | |
if (!inserter.nodeExists(agency)) { | |
Map<String, Object> properties = new HashMap<String, Object>(); | |
properties.put("Label", rs.getString("agencyName")); | |
inserter.createNode(agency, properties); | |
} | |
} | |
if (rs.getString("supplierABN") != "0") { | |
supplier = doHash(rs.getString("supplierABN")); | |
} else { | |
supplier = doHash(rs.getString("supplierName")); | |
} | |
// inject some data | |
if (!inserter.nodeExists(supplier)) { | |
Map<String, Object> properties = new HashMap<String, Object>(); | |
properties.put("Label", rs.getString("supplierName")); | |
inserter.createNode(supplier, properties); | |
} | |
inserter.createRelationship(agency, supplier, | |
DynamicRelationshipType.withName("KNOWS"), null); | |
} | |
// Close the result set, statement and the connection | |
rs.close(); | |
stmt.close(); | |
conn.close(); | |
} catch (SQLException se) { | |
System.out.println("SQL Exception:"); | |
// Loop through the SQL Exceptions | |
while (se != null) { | |
System.out.println("State : " + se.getSQLState()); | |
System.out.println("Message: " + se.getMessage()); | |
System.out.println("Error : " + se.getErrorCode()); | |
se = se.getNextException(); | |
} | |
} catch (Exception e) { | |
System.out.println(e); | |
} | |
// shutdown, makes sure all changes are written to disk | |
inserter.shutdown(); | |
} | |
static long doHash(String input) { | |
MessageDigest m; | |
try { | |
m = MessageDigest.getInstance("MD5"); | |
m.reset(); | |
m.update(input.getBytes()); | |
byte[] digest = m.digest(); | |
return new BigInteger(1, digest).longValue(); | |
} catch (NoSuchAlgorithmException e) { | |
// TODO Auto-generated catch block | |
e.printStackTrace(); | |
} | |
return 0; | |
} | |
} |
import java.sql.DriverManager; | |
import java.sql.Connection; | |
import java.sql.SQLException; | |
public class JDBCExample { | |
public static void main(String[] argv) { | |
System.out.println("-------- PostgreSQL " | |
+ "JDBC Connection Testing ------------"); | |
try { | |
Class.forName("org.postgresql.Driver"); | |
} catch (ClassNotFoundException e) { | |
System.out.println("Where is your PostgreSQL JDBC Driver? " | |
+ "Include in your library path!"); | |
e.printStackTrace(); | |
return; | |
} | |
System.out.println("PostgreSQL JDBC Driver Registered!"); | |
Connection connection = null; | |
try { | |
connection = DriverManager.getConnection( | |
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "postgres", | |
"snmc"); | |
} catch (SQLException e) { | |
System.out.println("Connection Failed! Check output console"); | |
e.printStackTrace(); | |
return; | |
} | |
if (connection != null) { | |
System.out.println("You made it, take control your database now!"); | |
} else { | |
System.out.println("Failed to make connection!"); | |
} | |
} | |
} |
Binary files /dev/null and b/admin/neo4jimporter/target/classes/Importer.class differ
Binary files /dev/null and b/admin/neo4jimporter/target/classes/JDBCExample.class differ
<?php | <?php |
include_once ("../../lib/common.inc.php"); | include_once ("../../lib/common.inc.php"); |
function processFile($fpath, $tablename) | function processFile($fpath, $tablename) |
{ | { |
global $conn; | global $conn; |
echo " ============== $fpath ============== <br>"; | echo " ============== $fpath ============== <br>"; |
flush(); | flush(); |
$row = 1; | $row = 1; |
$success = 0; | $success = 0; |
$dupes = 0; | $dupes = 0; |
$handle = fopen($fpath, "r"); | $handle = fopen($fpath, "r"); |
//"t" mode string translates windows line breaks to unix | //"t" mode string translates windows line breaks to unix |
$datamapping0507 = array( | $datamapping0507 = array( |
"Agency" => "agencyName", | "Agency" => "agencyName", |
"CN ID" => "CNID", | "CN ID" => "CNID", |
"Publish Date" => "publishDate", | "Publish Date" => "publishDate", |
"Contract Start Date" => "contractStart", | "Contract Start Date" => "contractStart", |
"Contract End Date" => "contractEnd", | "Contract End Date" => "contractEnd", |
"Value (AUD)" => "value", | "Value (AUD)" => "value", |
"Title" => "description", | "Title" => "description", |
"Category" => "category", | "Category" => "category", |
"ATM ID" => "atmID", | "ATM ID" => "atmID", |
"Supplier Name" => "supplierName", | "Supplier Name" => "supplierName", |
"LastUpdated" => "amendDate", | "LastUpdated" => "amendDate", |
"" => "" | "" => "" |
); | ); |
$headers; | $headers; |
$contractNoticeFields = array( | $contractNoticeFields = array( |
"importFile", | "importFile", |
"CNID", | "CNID", |
"description", | "description", |
"agencyName", | "agencyName", |
"publishDate", | "publishDate", |
"category", | "category", |
"contractStart", | "contractStart", |
"contractEnd", | "contractEnd", |
"value", | "value", |
"atmID", | "atmID", |
"supplierName", | "supplierName", |
"amendDate" | "amendDate" |
); | ); |
if ($tablename == "contractnotice") { | if ($tablename == "contractnotice") { |
$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( '; | $contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( '; |
foreach ($contractNoticeFields as $key => $f) { | foreach ($contractNoticeFields as $key => $f) { |
$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; | $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?"; |
} | } |
$contractNoticeInsertQ.= ");"; | $contractNoticeInsertQ.= ");"; |
$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); | $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ); |
} | } |
while (($data = fgetcsv($handle, 1000, "\t")) !== false) { | while (($data = fgetcsv($handle, 1000, "\t")) !== false) { |
$num = count($data); | $num = count($data); |
if ($row == 3) { | if ($row == 3) { |
$headers = $data; | $headers = $data; |
} | } |
elseif ($row > 3) { | elseif ($row > 3) { |
if ($num > count($datamapping0507)) { | if ($num > count($datamapping0507)) { |
die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . "data:" .$num. print_r($data ,true). "mapping:" . count($datamapping0507). print_r($datamapping0507 ,true)); | die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . "data:" .$num. print_r($data ,true). "mapping:" . count($datamapping0507). print_r($datamapping0507 ,true)); |
} | } |
$contractNoticeInsert = Array(); | $contractNoticeInsert = Array(); |
$contractNoticeInsert[] = $fpath; | $contractNoticeInsert[] = $fpath; |
$keys = array_keys($datamapping0507); | $keys = array_keys($datamapping0507); |
for ($c = 0; $c < $num; $c++) { | for ($c = 0; $c < $num; $c++) { |
$data[$c] = trim($data[$c], "="); | $data[$c] = trim($data[$c], "="); |
$data[$c] = trim($data[$c], "\""); | $data[$c] = trim($data[$c], "\""); |
if ($tablename == "contractnotice") { | if ($tablename == "contractnotice") { |
if (in_array(($datamapping0507[$headers[$c]]) , $contractNoticeFields)) { | if (in_array(($datamapping0507[$headers[$c]]) , $contractNoticeFields)) { |
if (($datamapping0507[$headers[$c]]) == "parentCN" || ($datamapping0507[$headers[$c]]) == "CNID") { | if (($datamapping0507[$headers[$c]]) == "parentCN" || ($datamapping0507[$headers[$c]]) == "CNID") { |
$data[$c] = substr($data[$c], 2); // take off the "CN" prefix | $data[$c] = substr($data[$c], 2); // take off the "CN" prefix |
$data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers | $data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers |
if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric"); | if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric"); |
if ($data[$c] > 0) { | if ($data[$c] > 0) { |
$contractNoticeInsert[] = $data[$c]; | $contractNoticeInsert[] = $data[$c]; |
} | } |
else { | else { |
$contractNoticeInsert[] = 0; | $contractNoticeInsert[] = 0; |
} | } |
} | } |
elseif (($datamapping0507[$headers[$c]]) == "supplierABN") { | elseif (($datamapping0507[$headers[$c]]) == "supplierABN") { |
if ($data[$c] > 0) { | if ($data[$c] > 0) { |
$contractNoticeInsert[] = $data[$c]; | $contractNoticeInsert[] = $data[$c]; |
} | } |
else { | else { |
$contr |