projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Fix CN frequency distribution graph
Fix CN frequency distribution graph

file:a/admin/neo4jimporter/pom.xml -> file:b/admin/neo4jimporter/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.lambdacomplex.contractdashboard</groupId> <groupId>org.lambdacomplex.contractdashboard</groupId>
<artifactId>neo4jimporter</artifactId> <artifactId>neo4jimporter</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId> <artifactId>neo4j</artifactId>
<version>1.6.1</version> <version>1.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>postgresql</groupId> <groupId>postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<version>9.0-801.jdbc4</version> <version>9.0-801.jdbc4</version>
</dependency> </dependency>
</dependencies> </dependencies>
</project> </project>
file:a/admin/neo4jimporter/src/main/java/Importer.java -> file:b/admin/neo4jimporter/src/main/java/Importer.java
   
import java.io.ObjectInputStream.GetField; import java.io.ObjectInputStream.GetField;
import java.math.BigInteger; import java.math.BigInteger;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.SQLWarning; import java.sql.SQLWarning;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
   
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.index.BatchInserterIndex; import org.neo4j.graphdb.index.BatchInserterIndex;
import org.neo4j.graphdb.index.BatchInserterIndexProvider; import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.helpers.collection.MapUtil; import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.kernel.impl.batchinsert.BatchInserter; import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl; import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
import org.neo4j.index.impl.lucene.*; import org.neo4j.index.impl.lucene.*;
   
public class Importer { public class Importer {
   
public static void main(String[] argv) { public static void main(String[] argv) {
BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert"); BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter); BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact")); BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
labels.setCacheCapacity("Label", 100000); labels.setCacheCapacity("Label", 100000);
   
   
   
System.out.println("-------- PostgreSQL " System.out.println("-------- PostgreSQL "
+ "JDBC Connection Testing ------------"); + "JDBC Connection Testing ------------");
   
try { try {
   
Class.forName("org.postgresql.Driver"); Class.forName("org.postgresql.Driver");
   
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
   
System.out.println("Where is your PostgreSQL JDBC Driver? " System.out.println("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!"); + "Include in your library path!");
e.printStackTrace(); e.printStackTrace();
   
} }
   
System.out.println("PostgreSQL JDBC Driver Registered!"); System.out.println("PostgreSQL JDBC Driver Registered!");
   
Connection conn = null; Connection conn = null;
   
try { try {
   
conn = DriverManager.getConnection( conn = DriverManager.getConnection(
"jdbc:postgresql://127.0.0.1:5432/contractDashboard", "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
"postgres", "snmc"); "postgres", "snmc");
   
} catch (SQLException e) { } catch (SQLException e) {
   
System.out.println("Connection Failed! Check output console"); System.out.println("Connection Failed! Check output console");
e.printStackTrace(); e.printStackTrace();
   
} }
   
if (conn != null) { if (conn != null) {
System.out.println("You made it, take control your database now!"); System.out.println("You made it, take control your database now!");
} else { } else {
System.out.println("Failed to make connection!"); System.out.println("Failed to make connection!");
} }
try { try {
// Print all warnings // Print all warnings
for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) { for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
System.out.println("SQL Warning:"); System.out.println("SQL Warning:");
System.out.println("State : " + warn.getSQLState()); System.out.println("State : " + warn.getSQLState());
System.out.println("Message: " + warn.getMessage()); System.out.println("Message: " + warn.getMessage());
System.out.println("Error : " + warn.getErrorCode()); System.out.println("Error : " + warn.getErrorCode());
} }
   
// Get a statement from the connection // Get a statement from the connection
Statement stmt = conn.createStatement(); Statement stmt = conn.createStatement();
   
// Execute the query // Execute the query
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", " ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum " + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'" + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
+ " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", " + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
+ " contractnotice.\"supplierABN\",contractnotice.\"supplierName\""); + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
String previousAgency = ""; String previousAgency = "";
GraphDatabaseService gds = inserter.getGraphDbService(); GraphDatabaseService gds = inserter.getGraphDbService();
HashMap<String, Long> supplierIDs = new HashMap<String, Long>(); HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
   
// Loop through the result set // Loop through the result set
while (rs.next()) { while (rs.next()) {
long supplierID, agencyID; long supplierID, agencyID;
String supplierKey; String supplierKey;
if (agencyIDs.get(rs.getString("agencyName")) == null) { if (agencyIDs.get(rs.getString("agencyName")) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("agencyName")); myNode.setProperty("Label", rs.getString("agencyName"));
myNode.setProperty("type", "agency"); myNode.setProperty("type", "agency");
agencyIDs.put(rs.getString("agencyName"), myNode.getId()); agencyIDs.put(rs.getString("agencyName"), myNode.getId());
if (myNode.getId() % 100 == 0) { if (myNode.getId() %100 == 0) {
System.out.println("Agency " + myNode.getId()); System.out.println("Agency "+myNode.getId());
} }
} }
agencyID = agencyIDs.get(rs.getString("agencyName")); agencyID = agencyIDs.get(rs.getString("agencyName"));
   
   
if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") { if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
supplierKey = rs.getString("supplierABN"); supplierKey = rs.getString("supplierABN");
} else { } else {
supplierKey = rs.getString("supplierName"); supplierKey = rs.getString("supplierName");
} }
// inject some data // inject some data
if (supplierIDs.get(supplierKey) == null) { if (supplierIDs.get(supplierKey) == null) {
Node myNode = gds.createNode(); Node myNode = gds.createNode();
myNode.setProperty("Label", rs.getString("supplierName")); myNode.setProperty("Label", rs.getString("supplierName"));
myNode.setProperty("type", "supplier"); myNode.setProperty("type", "supplier");
supplierIDs.put(supplierKey, myNode.getId()); supplierIDs.put(supplierKey, myNode.getId());
if (myNode.getId() % 1000 == 0) { if (myNode.getId() %1000 == 0) {
System.out.println("Supplier " + myNode.getId()); System.out.println("Supplier "+myNode.getId());
} }
} }
supplierID = supplierIDs.get(supplierKey); supplierID = supplierIDs.get(supplierKey);
   
   
long rel = inserter.createRelationship(agencyID, supplierID, long rel = inserter.createRelationship(agencyID, supplierID,
DynamicRelationshipType.withName("KNOWS"), null); DynamicRelationshipType.withName("KNOWS"), null);
inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum")); inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
   
} }
// Close the result set, statement and the connection // Close the result set, statement and the connection
rs.close(); rs.close();
stmt.close(); stmt.close();
conn.close(); conn.close();
} catch (SQLException se) { } catch (SQLException se) {
System.out.println("SQL Exception:"); System.out.println("SQL Exception:");
   
// Loop through the SQL Exceptions // Loop through the SQL Exceptions
while (se != null) { while (se != null) {
System.out.println("State : " + se.getSQLState()); System.out.println("State : " + se.getSQLState());
System.out.println("Message: " + se.getMessage()); System.out.println("Message: " + se.getMessage());
System.out.println("Error : " + se.getErrorCode()); System.out.println("Error : " + se.getErrorCode());
   
se = se.getNextException(); se = se.getNextException();
} }
} }
//make the changes visible for reading, use this sparsely, requires IO! //make the changes visible for reading, use this sparsely, requires IO!
labels.flush(); labels.flush();
   
// Make sure to shut down the index provider // Make sure to shut down the index provider
indexProvider.shutdown(); indexProvider.shutdown();
inserter.shutdown(); inserter.shutdown();
} }
   
   
} }
file:a/admin/neo4jimporter/target/classes/Importer.class -> file:b/admin/neo4jimporter/target/classes/Importer.class 
 Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
file:b/admin/partialdata/scraper.txt (new)
  <?php
  date_default_timezone_set('Australia/Melbourne');
  $split = false;
  function format_bytes($size) {
  $units = array(' B', ' KB', ' MB', ' GB', ' TB');
  for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
  return round($size, 2).$units[$i];
  }
 
  $days = 4;
  if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
  $startDate = strtotime("05-Jun-2008");
  if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
 
  function getFile($startDate, $days, $minVal, $maxVal) {
  global $split;
  $endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
  $file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
  echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
  $url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
  echo "<!-- $url -->";
  $current = file_get_contents($url);
  if (strpos($current,"There are no results that match your selection.")> 0 ) {
  echo "<font color=red>Empty file!</font><br>";
  }
  if (strpos($current,"Your search returned more than 1000 results.") === false) {
  file_put_contents($file, $current);
  echo "$file saved<br>";
  echo format_bytes(filesize($file))."<br>";
  echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
  echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
  echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
  flush();
  if (!isset($_REQUEST['split']) && !$split) {
  echo "Success so fetching next $days... <br>";
  getFile($endDate, $days, "" , "");
  }
  return true;
  } else {
  echo "<font color=red>Too many records!</font><br>";
  echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
  echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
  flush();
  if (!isset($_REQUEST['split']) && !$split) {
  echo "Failure so splitting ... <br>";
  doSplit($startDate, $days);
  }
  return false;
  }
  }
  function doSplit($startDate, $days) {
  global $split;
  $split = true;
  set_time_limit(20);
  getFile($startDate, $days, 0, 12000);
  getFile($startDate, $days, 12000, 16000);
  getFile($startDate, $days, 16000, 20000);
  getFile($startDate, $days, 20000, 30000);
  getFile($startDate, $days, 30000, 40000);
  // getFile($startDate, $days, 40000, 80000);
  getFile($startDate, $days, 40000, 60000);
  getFile($startDate, $days, 60000, 80000);
  // getFile($startDate, $days, 80000, 300000);
  getFile($startDate, $days, 80000, 150000);
  getFile($startDate, $days, 150000, 300000);
  getFile($startDate, $days, 300000, 999999999);
  }
  if (isset($_REQUEST['split'])) {
  doSplit($startDate, $days);
  } else {
  getFile($startDate, $days, "" , "");
  }
  ?>
 
file:a/displayCalendar.php -> file:b/displayCalendar.php
<?php <?php
include_once("./lib/common.inc.php"); include_once("./lib/common.inc.php");
include_header("Months and Years"); include_header("Months and Years");
if ($_REQUEST['month']) { if ($_REQUEST['month']) {
echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>"; echo "<center><h1>" . $_REQUEST['month'] . "</h1></center>";
$monthParts = explode("-", $_REQUEST['month']); $monthParts = explode("-", $_REQUEST['month']);
$query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName" $query = 'SELECT "CNID", "description", "value", "agencyName", "category", "contractStart", "supplierName"
FROM contractnotice FROM contractnotice
WHERE "childCN" is null WHERE "childCN" is null
AND extract(year from "contractStart") = :year AND extract(year from "contractStart") = :year
AND extract(month from "contractStart") = :month AND extract(month from "contractStart") = :month
ORDER BY value DESC'; ORDER BY value DESC';
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":month", $monthParts[0]); $query->bindParam(":month", $monthParts[0]);
$query->bindParam(":year", $monthParts[1]); $query->bindParam(":year", $monthParts[1]);
$query->execute(); $query->execute();
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
//MethodCountGraph($supplier); MethodCountGraph($supplier);
//CnCGraph($supplier); CnCGraph($supplier);
echo "<table> <thead> echo "<table> <thead>
<tr> <tr>
<th>Contract Notice Number</th> <th>Contract Notice Number</th>
<th>Contract Description</th> <th>Contract Description</th>
<th>Total Contract Value</th> <th>Total Contract Value</th>
<th>Agency</th> <th>Agency</th>
<th>Contract Start Date</th> <th>Contract Start Date</th>
<th>Supplier</th> <th>Supplier</th>
</tr> </tr>
</thead>"; </thead>";
foreach ($query->fetchAll() as $row) { foreach ($query->fetchAll() as $row) {
setlocale(LC_MONETARY, 'en_US'); setlocale(LC_MONETARY, 'en_US');
$value = number_format(doubleval($row['value']), 2); $value = number_format(doubleval($row['value']), 2);
echo ("<tr> echo ("<tr>
<td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td> <td><a href=\"displayContract.php?CNID={$row['CNID']}\">{$row['CNID']}</a></td>
<td><b>{$row['description']}</b></a></td> <td><b>{$row['description']}</b></a></td>
<td>\$$value</td><td>{$row['agencyName']}</td> <td>\$$value</td><td>{$row['agencyName']}</td>
<td>{$row['contractStart']}</td> <td>{$row['contractStart']}</td>
<td>{$row['supplierName']}</td> <td>{$row['supplierName']}</td>
</tr>"); </tr>");
} }
echo "</table>"; echo "</table>";
} else { } else {
/* /*
split by year split by year
todo: todo:
Year/Month drilldown - largest contracts, agencies, suppliers Year/Month drilldown - largest contracts, agencies, suppliers
count per month count per month
big picture graphs? */ big picture graphs? */
echo '<img src="graphs/displayContractStartingGraph.php">'; echo '<img src="graphs/displayContractStartingGraph.php">';
echo '<img src="graphs/displayContractPublishedGraph.php">'; echo '<img src="graphs/displayContractPublishedGraph.php">';