update overviewer and neo4j exports
update overviewer and neo4j exports

--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -7,12 +7,13 @@
   	<dependency>
   		<groupId>org.neo4j</groupId>
   		<artifactId>neo4j</artifactId>
-  		<version>1.8.RC1</version>
+  		<version>2.0.0-M03</version>
   	</dependency>
   	<dependency>
     <groupId>postgresql</groupId>
     <artifactId>postgresql</artifactId>
-    <version>9.0-801.jdbc4</version>
+    <version>9.1-901.jdbc4</version>
 </dependency>
   </dependencies>
 </project>
+

--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -5,25 +5,34 @@
 import java.sql.SQLException;

 import java.sql.SQLWarning;

 import java.sql.Statement;

+

+import java.io.File;

+import java.io.IOException;

+import java.io.InputStream;

+import java.io.Writer;

 import java.util.HashMap;

+import java.util.Map;

 

+import org.neo4j.graphdb.Direction;

+import org.neo4j.graphdb.DynamicLabel;

 import org.neo4j.graphdb.DynamicRelationshipType;

 import org.neo4j.graphdb.GraphDatabaseService;

+import org.neo4j.graphdb.Label;

 import org.neo4j.graphdb.Node;

-import org.neo4j.graphdb.index.BatchInserterIndex;

-import org.neo4j.graphdb.index.BatchInserterIndexProvider;

+import org.neo4j.graphdb.RelationshipType;

 import org.neo4j.helpers.collection.MapUtil;

-import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;

-import org.neo4j.kernel.impl.batchinsert.BatchInserter;

-import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;

+import org.neo4j.unsafe.batchinsert.BatchInserter;

+import org.neo4j.unsafe.batchinsert.BatchInserters;

 

 public class Importer {

 

     public static void main(String[] argv) {

-        BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");

-        BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);

-        BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));

-        labels.setCacheCapacity("Label", 100000);

+Map<String, String> config = new HashMap<String, String>();

+config.put( "neostore.nodestore.db.mapped_memory", "90M" );

+BatchInserter inserter = BatchInserters.inserter("target/batchinserter-example-config", config );

+        //BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);

+        //BatchInserterIndex names = indexProvider.nodeIndex("names", MapUtil.stringMap("type", "exact"));

+        //names.setCacheCapacity("name", 100000);

 

 

 

@@ -78,52 +87,56 @@
 

             // Execute the query

             ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "

-                    + "  contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "

+                    + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END) as supplierID , max(contractnotice.\"supplierName\") as \"supplierName\",sum(value) as sum "

                     + "FROM  public.contractnotice  GROUP BY contractnotice.\"agencyName\", "

-                    + "  contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");

-            String previousAgency = "";

-            GraphDatabaseService gds = inserter.getGraphDbService();

+                    + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END)");

             HashMap<String, Long> supplierIDs = new HashMap<String, Long>();

             HashMap<String, Long> agencyIDs = new HashMap<String, Long>();

+

+Label agencyLabel = DynamicLabel.label( "Agency" );

+inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" );

+Label supplierLabel = DynamicLabel.label( "Supplier" );

+inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" );

 

             // Loop through the result set

             while (rs.next()) {

                 long supplierID, agencyID;

                 String supplierKey;

                 if (agencyIDs.get(rs.getString("agencyName")) == null) {

-                    Node myNode = gds.createNode();

-                    myNode.setProperty("Label", rs.getString("agencyName"));

-                    myNode.setProperty("type", "agency");

-                    agencyIDs.put(rs.getString("agencyName"), myNode.getId());

-                    if (myNode.getId() % 100 == 0) {

-                        System.out.println("Agency " + myNode.getId());

+		    Map<String, Object> properties = new HashMap<String, Object>();

+                    properties.put("name", rs.getString("agencyName"));

+                    properties.put("type", rs.getString("agency"));

+		    agencyID = inserter.createNode(properties, agencyLabel);

+                    agencyIDs.put(rs.getString("agencyName"), agencyID);

+                    if (agencyID % 10 == 0) {

+                        System.out.println("Agency " + agencyID);

                     }

                 }

                 agencyID = agencyIDs.get(rs.getString("agencyName"));

 

 

-                if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {

-                    supplierKey = rs.getString("supplierABN");

-                } else {

-                    supplierKey = rs.getString("supplierName");

-                }

                 // inject some data 

-                if (supplierIDs.get(supplierKey) == null) {

-                    Node myNode = gds.createNode();

-                    myNode.setProperty("Label", rs.getString("supplierName"));

-                    myNode.setProperty("type", "supplier");

-                    supplierIDs.put(supplierKey, myNode.getId());

-                    if (myNode.getId() % 1000 == 0) {

-                        System.out.println("Supplier " + myNode.getId());

+                if (supplierIDs.get(rs.getString("supplierID")) == null) {

+		    Map<String, Object> properties = new HashMap<String, Object>();

+                    properties.put("name", rs.getString("supplierName"));

+                    properties.put("type", rs.getString("supplier"));

+		    supplierID = inserter.createNode(properties, supplierLabel);

+                    supplierIDs.put(rs.getString("supplierID"), supplierID);

+                    if (supplierID % 1000 == 0) {

+                        System.out.println("Supplier " + supplierID);

                     }

                 }

-                supplierID = supplierIDs.get(supplierKey);

+                supplierID = supplierIDs.get(rs.getString("supplierID"));

 

 

-                long rel = inserter.createRelationship(agencyID, supplierID,

-                        DynamicRelationshipType.withName("KNOWS"), null);

-                inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));

-

+// To set properties on the relationship, use a properties map

+// instead of null as the last parameter.

+Map<String, Object> properties = new HashMap<String, Object>();

+properties.put( "value", rs.getDouble("sum"));

+                inserter.createRelationship(agencyID, supplierID,

+                        DynamicRelationshipType.withName("PAYS"), properties);

+                inserter.createRelationship(supplierID, agencyID,

+                        DynamicRelationshipType.withName("PAID_BY"), properties);

             }

             // Close the result set, statement and the connection

             rs.close();

@@ -142,10 +155,11 @@
             }

         }

 //make the changes visible for reading, use this sparsely, requires IO!

-        labels.flush();

+//        names.flush();

 

 // Make sure to shut down the index provider

-        indexProvider.shutdown();

+//        indexProvider.shutdown();

         inserter.shutdown();

     }

-}
+}

+

 Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
--- a/exportOverview.csv.php
+++ b/exportOverview.csv.php
@@ -11,10 +11,11 @@
     $unspsc[$row['UNSPSC']] = $row['Title'];
 }
 
+//(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url 
 $query = $conn->prepare('
-SELECT "CNID" as uid, description as text,
-(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url from "contractnotice"
-where "childCN" is null'
+SELECT "CNID" as id, description as text
+from "contractnotice"
+where "childCN" is null limit 10000'
         , array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
 $query->execute();
 $errors = $conn->errorInfo();