<?xml version="1.0" encoding="UTF-8"?> | |
<classpath> | |
<classpathentry kind="src" output="target/classes" path="src/main/java"/> | |
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/> | |
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/> | |
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/> | |
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/> | |
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/> | |
<classpathentry kind="output" path="target/classes"/> | |
</classpath> | |
<?xml version="1.0" encoding="UTF-8"?> | |
<projectDescription> | |
<name>neo4jimporter</name> | |
<comment></comment> | |
<projects> | |
</projects> | |
<buildSpec> | |
<buildCommand> | |
<name>org.eclipse.jdt.core.javabuilder</name> | |
<arguments> | |
</arguments> | |
</buildCommand> | |
<buildCommand> | |
<name>org.eclipse.m2e.core.maven2Builder</name> | |
<arguments> | |
</arguments> | |
</buildCommand> | |
</buildSpec> | |
<natures> | |
<nature>org.eclipse.jdt.core.javanature</nature> | |
<nature>org.eclipse.m2e.core.maven2Nature</nature> | |
</natures> | |
</projectDescription> | |
#Sun Aug 07 18:15:32 EST 2011 | |
eclipse.preferences.version=1 | |
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 | |
org.eclipse.jdt.core.compiler.compliance=1.5 | |
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning | |
org.eclipse.jdt.core.compiler.source=1.5 | |
#Sun Aug 07 18:14:30 EST 2011 | |
activeProfiles= | |
eclipse.preferences.version=1 | |
resolveWorkspaceProjects=true | |
version=1 | |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |
<modelVersion>4.0.0</modelVersion> | |
<groupId>org.lambdacomplex.contractdashboard</groupId> | |
<artifactId>neo4jimporter</artifactId> | |
<version>0.0.1-SNAPSHOT</version> | |
<dependencies> | |
<dependency> | |
<groupId>org.neo4j</groupId> | |
<artifactId>neo4j-kernel</artifactId> | |
<version>1.4</version> | |
</dependency> | |
<dependency> | |
<groupId>postgresql</groupId> | |
<artifactId>postgresql</artifactId> | |
<version>9.0-801.jdbc4</version> | |
</dependency> | |
</dependencies> | |
</project> |
import java.io.ObjectInputStream.GetField; | |
import java.math.BigInteger; | |
import java.security.MessageDigest; | |
import java.security.NoSuchAlgorithmException; | |
import java.sql.Connection; | |
import java.sql.DriverManager; | |
import java.sql.ResultSet; | |
import java.sql.SQLException; | |
import java.sql.SQLWarning; | |
import java.sql.Statement; | |
import java.util.HashMap; | |
import java.util.Map; | |
import org.neo4j.graphdb.DynamicRelationshipType; | |
import org.neo4j.kernel.impl.batchinsert.BatchInserter; | |
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; | |
public class Importer { | |
public static void main(String[] argv) { | |
Map<String,String> props = new HashMap<String, String>(); | |
props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes> | |
props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes> | |
// create the batch inserter | |
BatchInserter inserter = new | |
BatchInserterImpl("neo4j-db/",props | |
); | |
System.out.println("-------- PostgreSQL " | |
+ "JDBC Connection Testing ------------"); | |
try { | |
Class.forName("org.postgresql.Driver"); | |
} catch (ClassNotFoundException e) { | |
System.out.println("Where is your PostgreSQL JDBC Driver? " | |
+ "Include in your library path!"); | |
e.printStackTrace(); | |
} | |
System.out.println("PostgreSQL JDBC Driver Registered!"); | |
Connection conn = null; | |
try { | |
conn = DriverManager.getConnection( | |
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", | |
"postgres", "snmc"); | |
} catch (SQLException e) { | |
System.out.println("Connection Failed! Check output console"); | |
e.printStackTrace(); | |
} | |
if (conn != null) { | |
System.out.println("You made it, take control your database now!"); | |
} else { | |
System.out.println("Failed to make connection!"); | |
} | |
try { | |
// Print all warnings | |
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn | |
.getNextWarning()) { | |
System.out.println("SQL Warning:"); | |
System.out.println("State : " + warn.getSQLState()); | |
System.out.println("Message: " + warn.getMessage()); | |
System.out.println("Error : " + warn.getErrorCode()); | |
} | |
// Get a statement from the connection | |
Statement stmt = conn.createStatement(); | |
// Execute the query | |
ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;"); | |
String previousAgency = ""; | |
// Loop through the result set | |
while (rs.next()) { | |
long supplier,agency; | |
agency = doHash(rs.getString("agencyName")); | |
if (rs.getString("agencyName") != previousAgency) { | |
if (!inserter.nodeExists(agency)) { | |
Map<String, Object> properties = new HashMap<String, Object>(); | |
properties.put("Label", rs.getString("agencyName")); | |
inserter.createNode(agency, properties); | |
} | |
} | |
if (rs.getString("supplierABN") != "0") { | |
supplier = doHash(rs.getString("supplierABN")); | |
} else { | |
supplier = doHash(rs.getString("supplierName")); | |
} | |
// inject some data | |
if (!inserter.nodeExists(supplier)) { | |
Map<String, Object> properties = new HashMap<String, Object>(); | |
properties.put("Label", rs.getString("supplierName")); | |
inserter.createNode(supplier, properties); | |
} | |
inserter.createRelationship(agency, supplier, | |
DynamicRelationshipType.withName("KNOWS"), null); | |
} | |
// Close the result set, statement and the connection | |
rs.close(); | |
stmt.close(); | |
conn.close(); | |
} catch (SQLException se) { | |
System.out.println("SQL Exception:"); | |
// Loop through the SQL Exceptions | |
while (se != null) { | |
System.out.println("State : " + se.getSQLState()); | |
System.out.println("Message: " + se.getMessage()); | |
System.out.println("Error : " + se.getErrorCode()); | |
se = se.getNextException(); | |
} | |
} catch (Exception e) { | |
System.out.println(e); | |
} | |
// shutdown, makes sure all changes are written to disk | |
inserter.shutdown(); | |
} | |
static long doHash(String input) { | |
MessageDigest m; | |
try { | |
m = MessageDigest.getInstance("MD5"); | |
m.reset(); | |
m.update(input.getBytes()); | |
byte[] digest = m.digest(); | |
return new BigInteger(1, digest).longValue(); | |
} catch (NoSuchAlgorithmException e) { | |
// TODO Auto-generated catch block | |
e.printStackTrace(); | |
} | |
return 0; | |
} | |
} |
import java.sql.DriverManager; | |
import java.sql.Connection; | |
import java.sql.SQLException; | |
public class JDBCExample { | |
public static void main(String[] argv) { | |
System.out.println("-------- PostgreSQL " | |
+ "JDBC Connection Testing ------------"); | |
try { | |
Class.forName("org.postgresql.Driver"); | |
} catch (ClassNotFoundException e) { | |
System.out.println("Where is your PostgreSQL JDBC Driver? " | |
+ "Include in your library path!"); | |
e.printStackTrace(); | |
return; | |
} | |
System.out.println("PostgreSQL JDBC Driver Registered!"); | |
Connection connection = null; | |
try { | |
connection = DriverManager.getConnection( | |
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "postgres", | |
"snmc"); | |
} catch (SQLException e) { | |
System.out.println("Connection Failed! Check output console"); | |
e.printStackTrace(); | |
return; | |
} | |
if (connection != null) { | |
System.out.println("You made it, take control your database now!"); | |
} else { | |
System.out.println("Failed to make connection!"); | |
} | |
} | |
} |
Binary files /dev/null and b/admin/neo4jimporter/target/classes/Importer.class differ
Binary files /dev/null and b/admin/neo4jimporter/target/classes/JDBCExample.class differ
<?php | |
include_once ("../../lib/common.inc.php"); | |
/* | |
update contractnotice set "supplierABN" = a."supplierABN" | |
from contractnotice as cn inner join (select "supplierABN", | |
"supplierName" from contractnotice where "supplierABN" | |
IS NOT NULL and "supplierABN" != 0) as a on | |
cn."supplierName" = a."supplierName" where | |
cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN" | |
IS NULL or contractnotice."supplierABN" = 0) */ | |
// http://www.lastcraft.com/browser_documentation.php | |
// http://code.google.com/p/phpquery/ | |
require('phpQuery-onefile.php'); | |
function getURL($url) { | |
//return file_get_contents($url); | |
$ch = curl_init($url); | |
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); | |
curl_setopt($ch, CURLOPT_HEADER, 0); | |
curl_setopt($ch, CURLOPT_TIMEOUT, 45); | |
$page = curl_exec($ch); | |
if (curl_errno($ch)) { | |
echo "<font color=red> Database temporarily unavailable: "; | |
echo curl_errno($ch) . " " . curl_error($ch); | |
echo $url; | |
echo "</font><br>"; | |
} | |
curl_close($ch); | |
return $page; | |
} | |
function getTextFromTHNode($Node, $Text = "") { | |
if ($Node->tagName == null) | |
return $Text.$Node->textContent; | |
if ($Node->tagName != "td") { | |
$Node = $Node->firstChild; | |
if ($Node != null) | |
$Text = getTextFromTHNode($Node, $Text); | |
while($Node->nextSibling != null) { | |
$Text = getTextFromTHNode($Node->nextSibling, $Text); | |
$Node = $Node->nextSibling; | |
} | |
} | |
return $Text; | |
} | |
function getTextFromNode($Node, $Text = "") { | |
if ($Node->tagName == null) | |
return $Text.$Node->textContent; | |
if ($Node->tagName != "th" && $Node->tagName != "span") { | |
$Node = $Node->firstChild; | |
if ($Node != null) | |
$Text = getTextFromNode($Node, $Text); | |
while($Node->nextSibling != null) { | |
$Text = getTextFromNode($Node->nextSibling, $Text); | |
$Node = $Node->nextSibling; | |
} | |
} | |
return $Text; | |
} | |
function dom_to_array($root) | |
{ | |
$result = array(); | |
if ($root->hasAttributes()) | |
{ | |
$attrs = $root->attributes; | |
foreach ($attrs as $i => $attr) | |
$result[$attr->name] = $attr->value; | |
} | |
$children = $root->childNodes; | |
if ($root->childNodes) { | |
if ($children->length == 1) | |
{ | |
$child = $children->item(0); | |
if ($child->nodeType == XML_TEXT_NODE) | |
{ | |
$result['_value'] = $child->nodeValue; | |
if (count($result) == 1) | |
return $result['_value']; | |
else | |
return $result; | |
} | |
} | |
$group = array(); | |
for($i = 0; $i < $children->length; $i++) | |
{ | |
$child = $children->item($i); | |
if (!isset($result[$child->nodeName])) | |
$result[$child->nodeName] = dom_to_array($child); | |
else | |
{ | |
if (!isset($group[$child->nodeName])) | |
{ | |
$tmp = $result[$child->nodeName]; | |
$result[$child->nodeName] = array($tmp); | |
$group[$child->nodeName] = 1; | |
} | |
$result[$child->nodeName][] = dom_to_array($child); | |
} | |
} | |
} | |
return $result; | |
} | |
function importCN($cnid) { | |
global $conn; | |
$CN = str_replace("-A", "00",$cnid); | |
// check if already complete | |