projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Port data exporter to postgres
Port data exporter to postgres

file:b/admin/neo4jimporter/.classpath (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <classpath>
  <classpathentry kind="src" output="target/classes" path="src/main/java"/>
  <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
  <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
  <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
  <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
  <classpathentry kind="output" path="target/classes"/>
  </classpath>
 
file:b/admin/neo4jimporter/.project (new)
  <?xml version="1.0" encoding="UTF-8"?>
  <projectDescription>
  <name>neo4jimporter</name>
  <comment></comment>
  <projects>
  </projects>
  <buildSpec>
  <buildCommand>
  <name>org.eclipse.jdt.core.javabuilder</name>
  <arguments>
  </arguments>
  </buildCommand>
  <buildCommand>
  <name>org.eclipse.m2e.core.maven2Builder</name>
  <arguments>
  </arguments>
  </buildCommand>
  </buildSpec>
  <natures>
  <nature>org.eclipse.jdt.core.javanature</nature>
  <nature>org.eclipse.m2e.core.maven2Nature</nature>
  </natures>
  </projectDescription>
 
file:b/admin/neo4jimporter/.settings/org.eclipse.jdt.core.prefs (new)
  #Sun Aug 07 18:15:32 EST 2011
  eclipse.preferences.version=1
  org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
  org.eclipse.jdt.core.compiler.compliance=1.5
  org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
  org.eclipse.jdt.core.compiler.source=1.5
 
file:b/admin/neo4jimporter/.settings/org.eclipse.m2e.core.prefs (new)
  #Sun Aug 07 18:14:30 EST 2011
  activeProfiles=
  eclipse.preferences.version=1
  resolveWorkspaceProjects=true
  version=1
 
file:b/admin/neo4jimporter/pom.xml (new)
  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>org.lambdacomplex.contractdashboard</groupId>
  <artifactId>neo4jimporter</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <dependencies>
  <dependency>
  <groupId>org.neo4j</groupId>
  <artifactId>neo4j-kernel</artifactId>
  <version>1.4</version>
  </dependency>
  <dependency>
  <groupId>postgresql</groupId>
  <artifactId>postgresql</artifactId>
  <version>9.0-801.jdbc4</version>
  </dependency>
  </dependencies>
  </project>
file:b/admin/neo4jimporter/src/main/java/Importer.java (new)
  import java.io.ObjectInputStream.GetField;
  import java.math.BigInteger;
  import java.security.MessageDigest;
  import java.security.NoSuchAlgorithmException;
  import java.sql.Connection;
  import java.sql.DriverManager;
  import java.sql.ResultSet;
  import java.sql.SQLException;
  import java.sql.SQLWarning;
  import java.sql.Statement;
  import java.util.HashMap;
  import java.util.Map;
 
  import org.neo4j.graphdb.DynamicRelationshipType;
  import org.neo4j.kernel.impl.batchinsert.BatchInserter;
  import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
 
  public class Importer {
 
  public static void main(String[] argv) {
 
  Map<String,String> props = new HashMap<String, String>();
  props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes>
  props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes>
  // create the batch inserter
  BatchInserter inserter = new
  BatchInserterImpl("neo4j-db/",props
  );
 
 
 
 
 
  System.out.println("-------- PostgreSQL "
  + "JDBC Connection Testing ------------");
 
  try {
 
  Class.forName("org.postgresql.Driver");
 
  } catch (ClassNotFoundException e) {
 
  System.out.println("Where is your PostgreSQL JDBC Driver? "
  + "Include in your library path!");
  e.printStackTrace();
 
  }
 
  System.out.println("PostgreSQL JDBC Driver Registered!");
 
  Connection conn = null;
 
  try {
 
  conn = DriverManager.getConnection(
  "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
  "postgres", "snmc");
 
  } catch (SQLException e) {
 
  System.out.println("Connection Failed! Check output console");
  e.printStackTrace();
 
  }
 
  if (conn != null) {
  System.out.println("You made it, take control your database now!");
  } else {
  System.out.println("Failed to make connection!");
  }
  try {
  // Print all warnings
  for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn
  .getNextWarning()) {
  System.out.println("SQL Warning:");
  System.out.println("State : " + warn.getSQLState());
  System.out.println("Message: " + warn.getMessage());
  System.out.println("Error : " + warn.getErrorCode());
  }
 
  // Get a statement from the connection
  Statement stmt = conn.createStatement();
 
  // Execute the query
  ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;");
  String previousAgency = "";
 
  // Loop through the result set
  while (rs.next()) {
  long supplier,agency;
  agency = doHash(rs.getString("agencyName"));
  if (rs.getString("agencyName") != previousAgency) {
  if (!inserter.nodeExists(agency)) {
  Map<String, Object> properties = new HashMap<String, Object>();
  properties.put("Label", rs.getString("agencyName"));
  inserter.createNode(agency, properties);
  }
  }
  if (rs.getString("supplierABN") != "0") {
  supplier = doHash(rs.getString("supplierABN"));
  } else {
  supplier = doHash(rs.getString("supplierName"));
  }
  // inject some data
  if (!inserter.nodeExists(supplier)) {
  Map<String, Object> properties = new HashMap<String, Object>();
 
  properties.put("Label", rs.getString("supplierName"));
  inserter.createNode(supplier, properties);
  }
 
  inserter.createRelationship(agency, supplier,
  DynamicRelationshipType.withName("KNOWS"), null);
 
  }
  // Close the result set, statement and the connection
  rs.close();
  stmt.close();
  conn.close();
  } catch (SQLException se) {
  System.out.println("SQL Exception:");
 
  // Loop through the SQL Exceptions
  while (se != null) {
  System.out.println("State : " + se.getSQLState());
  System.out.println("Message: " + se.getMessage());
  System.out.println("Error : " + se.getErrorCode());
 
  se = se.getNextException();
  }
  } catch (Exception e) {
  System.out.println(e);
  }
  // shutdown, makes sure all changes are written to disk
  inserter.shutdown();
  }
 
  static long doHash(String input) {
  MessageDigest m;
  try {
  m = MessageDigest.getInstance("MD5");
  m.reset();
  m.update(input.getBytes());
  byte[] digest = m.digest();
  return new BigInteger(1, digest).longValue();
  } catch (NoSuchAlgorithmException e) {
  // TODO Auto-generated catch block
  e.printStackTrace();
  }
  return 0;
 
  }
 
 
  }
file:b/admin/neo4jimporter/src/main/java/JDBCExample.java (new)
  import java.sql.DriverManager;
  import java.sql.Connection;
  import java.sql.SQLException;
 
  public class JDBCExample {
 
  public static void main(String[] argv) {
 
  System.out.println("-------- PostgreSQL "
  + "JDBC Connection Testing ------------");
 
  try {
 
  Class.forName("org.postgresql.Driver");
 
  } catch (ClassNotFoundException e) {
 
  System.out.println("Where is your PostgreSQL JDBC Driver? "
  + "Include in your library path!");
  e.printStackTrace();
  return;
 
  }
 
  System.out.println("PostgreSQL JDBC Driver Registered!");
 
  Connection connection = null;
 
  try {
 
  connection = DriverManager.getConnection(
  "jdbc:postgresql://127.0.0.1:5432/contractDashboard", "postgres",
  "snmc");
 
  } catch (SQLException e) {
 
  System.out.println("Connection Failed! Check output console");
  e.printStackTrace();
  return;
 
  }
 
  if (connection != null) {
  System.out.println("You made it, take control your database now!");
  } else {
  System.out.println("Failed to make connection!");
  }
  }
 
  }
file:b/admin/neo4jimporter/target/classes/Importer.class (new) 
 Binary files /dev/null and b/admin/neo4jimporter/target/classes/Importer.class differ
file:b/admin/neo4jimporter/target/classes/JDBCExample.class (new) 
 Binary files /dev/null and b/admin/neo4jimporter/target/classes/JDBCExample.class differ
file:a/admin/partialdata/scrapesingle.php -> file:b/admin/partialdata/scrapesingle.php
<?php <?php
  include_once ("../../lib/common.inc.php");
   
$cnid = 1234; $cnid = 1234;
// http://www.lastcraft.com/browser_documentation.php // http://www.lastcraft.com/browser_documentation.php
// http://code.google.com/p/phpquery/ // http://code.google.com/p/phpquery/
require('phpQuery-onefile.php'); require('phpQuery-onefile.php');
function dom_to_array($root) function dom_to_array($root)
{ {
$result = array(); $result = array();
   
if ($root->hasAttributes()) if ($root->hasAttributes())
{ {
$attrs = $root->attributes; $attrs = $root->attributes;
   
foreach ($attrs as $i => $attr) foreach ($attrs as $i => $attr)
$result[$attr->name] = $attr->value; $result[$attr->name] = $attr->value;
} }
   
$children = $root->childNodes; $children = $root->childNodes;
if ($root->childNodes) { if ($root->childNodes) {
if ($children->length == 1) if ($children->length == 1)
{ {
$child = $children->item(0); $child = $children->item(0);
   
if ($child->nodeType == XML_TEXT_NODE) if ($child->nodeType == XML_TEXT_NODE)
{ {
$result['_value'] = $child->nodeValue; $result['_value'] = $child->nodeValue;
   
if (count($result) == 1) if (count($result) == 1)
return $result['_value']; return $result['_value'];
else else
return $result; return $result;
} }
} }
   
$group = array(); $group = array();
   
for($i = 0; $i < $children->length; $i++) for($i = 0; $i < $children->length; $i++)
{ {
$child = $children->item($i); $child = $children->item($i);
   
if (!isset($result[$child->nodeName])) if (!isset($result[$child->nodeName]))
$result[$child->nodeName] = dom_to_array($child); $result[$child->nodeName] = dom_to_array($child);
else else
{ {
if (!isset($group[$child->nodeName])) if (!isset($group[$child->nodeName]))
{ {
$tmp = $result[$child->nodeName]; $tmp = $result[$child->nodeName];
$result[$child->nodeName] = array($tmp); $result[$child->nodeName] = array($tmp);
$group[$child->nodeName] = 1; $group[$child->nodeName] = 1;
} }
   
$result[$child->nodeName][] = dom_to_array($child); $result[$child->nodeName][] = dom_to_array($child);
} }
} }
} }
   
return $result; return $result;
} }
   
$site = "https://www.tenders.gov.au/"; $site = "https://www.tenders.gov.au/";
//$cn = phpQuery::newDocument(file_get_contents("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid)); //$cn = phpQuery::newDocument(file_get_contents("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
$searchResult = phpQuery::newDocument('<table class="four-col"> $searchResult = phpQuery::newDocument('<table class="four-col">
<tbody><tr> <tbody><tr>
<th>CN ID</th> <th>CN ID</th>
<td><a href="/?event=public.cn.view&amp;CNUUID=BB3B13EA-A700-39A5-F003088ACF798AF9">CN1234</a></td> <td><a href="/?event=public.cn.view&amp;CNUUID=BB3B13EA-A700-39A5-F003088ACF798AF9">CN1234</a></td>
<th> <th>
</th><td> </th><td>
</td></tr> </td></tr>
<tr> <tr>
<th>Agency</th> <th>Agency</th>
<td colspan="3">Department of Veterans Affairs</td> <td colspan="3">Department of Veterans Affairs</td>
</tr> </tr>
<tr> <tr>
<th>Publish Date</th> <th>Publish Date</th>
<td colspan="3"> 18-Aug-2006 </td> <td colspan="3"> 18-Aug-2006 </td>
</tr> </tr>
<tr> <tr>
<th>Category</th> <th>Category</th>
<td colspan="3">Computer services</td> <td colspan="3">Computer services</td>
</tr> </tr>
<tr> <tr>
<th>Contract Period</th> <th>Contract Period</th>
<td colspan="3"> 1-Aug-2006 to 31-Jul-2007 </td> <td colspan="3"> 1-Aug-2006 to 31-Jul-2007 </td>
</tr> </tr>
<tr> <tr>
<th>Contract Value (AUD)</th> <th>Contract Value (AUD)</th>
<td colspan="3"> $156,200.00 </td> <td colspan="3"> $156,200.00 </td>
</tr> </tr>
<tr> <tr>
<th>Supplier Name</th> <th>Supplier Name</th>
<td colspan="3">WIZARD INFORMATION SERVICES PTY LTD</td> <td colspan="3">WIZARD INFORMATION SERVICES PTY LTD</td>
</tr> </tr>
<tr> <tr>
<th></th> <th></th>
<td colspan="3"> <td colspan="3">
<p> <p>