From: Alexander Sadleir <maxious@lambdacomplex.org>
Date: Tue, 15 Nov 2011 07:50:54 +0000
Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=contractdashboard.git&a=commitdiff&h=e7a65556b17654f82887e0597999a58bbde3588d
---
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
---


--- a/admin/import.php
+++ b/admin/import.php
@@ -261,10 +261,10 @@
 }
 // run post import data processing
 // cn
-//include ("linkAmendments.php");
+include ("linkAmendments.php");
 include ("updateUNSPSC.php");
 // agency
-include ("setAgencyStatus.php");
-include ("setAgencyURLABN.php");
+//include ("setAgencyStatus.php");
+//include ("setAgencyURLABN.php");
 ?>
 

--- /dev/null
+++ b/admin/neo4jimporter/nbactions.xml
@@ -1,1 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>run</actionName>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+            </goals>
+            <properties>
+                <exec.classpathScope>runtime</exec.classpathScope>
+                <exec.args>-Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>debug</actionName>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+            </goals>
+            <properties>
+                <exec.classpathScope>runtime</exec.classpathScope>
+                <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+                <jpda.listen>true</jpda.listen>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>profile</actionName>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>${profiler.args} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+                <profiler.action>profile</profiler.action>
+                <exec.executable>${profiler.java}</exec.executable>
+            </properties>
+        </action>
+        <action>
+            <actionName>run</actionName>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+            </goals>
+            <properties>
+                <exec.classpathScope>runtime</exec.classpathScope>
+                <exec.args>-classpath %classpath Importer</exec.args>
+                <exec.executable>java</exec.executable>
+            </properties>
+        </action>
+    </actions>
 

--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -6,8 +6,8 @@
   <dependencies>
   	<dependency>
   		<groupId>org.neo4j</groupId>
-  		<artifactId>neo4j-kernel</artifactId>
-  		<version>1.4</version>
+  		<artifactId>neo4j</artifactId>
+  		<version>1.5</version>
   	</dependency>
   	<dependency>
     <groupId>postgresql</groupId>

--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -1,3 +1,4 @@
+
 import java.io.ObjectInputStream.GetField;
 import java.math.BigInteger;
 import java.security.MessageDigest;
@@ -12,144 +13,147 @@
 import java.util.Map;
 
 import org.neo4j.graphdb.DynamicRelationshipType;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.Node;
+import org.neo4j.graphdb.index.BatchInserterIndex;
+import org.neo4j.graphdb.index.BatchInserterIndexProvider;
+import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.kernel.impl.batchinsert.BatchInserter;
 import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
+import org.neo4j.index.impl.lucene.*;
 
 public class Importer {
 
-	public static void main(String[] argv) {
-		
-		 Map<String,String> props = new HashMap<String, String>();
-		 props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes>
-		 props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes>
-		  // create the batch inserter 
-		BatchInserter inserter = new
-		  BatchInserterImpl("neo4j-db/",props
-		 );
-		  
+    public static void main(String[] argv) {
+        BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
+        BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
+        BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
+        labels.setCacheCapacity("Label", 100000);
 
 
-		 
-		
-		System.out.println("-------- PostgreSQL "
-				+ "JDBC Connection Testing ------------");
 
-		try {
+        System.out.println("-------- PostgreSQL "
+                + "JDBC Connection Testing ------------");
 
-			Class.forName("org.postgresql.Driver");
+        try {
 
-		} catch (ClassNotFoundException e) {
+            Class.forName("org.postgresql.Driver");
 
-			System.out.println("Where is your PostgreSQL JDBC Driver? "
-					+ "Include in your library path!");
-			e.printStackTrace();
+        } catch (ClassNotFoundException e) {
 
-		}
+            System.out.println("Where is your PostgreSQL JDBC Driver? "
+                    + "Include in your library path!");
+            e.printStackTrace();
 
-		System.out.println("PostgreSQL JDBC Driver Registered!");
+        }
 
-		Connection conn = null;
+        System.out.println("PostgreSQL JDBC Driver Registered!");
 
-		try {
+        Connection conn = null;
 
-			conn = DriverManager.getConnection(
-					"jdbc:postgresql://127.0.0.1:5432/contractDashboard",
-					"postgres", "snmc");
+        try {
 
-		} catch (SQLException e) {
+            conn = DriverManager.getConnection(
+                    "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
+                    "postgres", "snmc");
 
-			System.out.println("Connection Failed! Check output console");
-			e.printStackTrace();
+        } catch (SQLException e) {
 
-		}
+            System.out.println("Connection Failed! Check output console");
+            e.printStackTrace();
 
-		if (conn != null) {
-			System.out.println("You made it, take control your database now!");
-		} else {
-			System.out.println("Failed to make connection!");
-		}
-		try {
-			// Print all warnings
-			for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn
-					.getNextWarning()) {
-				System.out.println("SQL Warning:");
-				System.out.println("State  : " + warn.getSQLState());
-				System.out.println("Message: " + warn.getMessage());
-				System.out.println("Error  : " + warn.getErrorCode());
-			}
+        }
 
-			// Get a statement from the connection
-			Statement stmt = conn.createStatement();
+        if (conn != null) {
+            System.out.println("You made it, take control your database now!");
+        } else {
+            System.out.println("Failed to make connection!");
+        }
+        try {
+            // Print all warnings
+            for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
+                System.out.println("SQL Warning:");
+                System.out.println("State  : " + warn.getSQLState());
+                System.out.println("Message: " + warn.getMessage());
+                System.out.println("Error  : " + warn.getErrorCode());
+            }
 
-			// Execute the query
-			ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\",   contractnotice.\"supplierABN\",   contractnotice.\"supplierName\" FROM  public.contractnotice limit 300;");
-String previousAgency = "";
+            // Get a statement from the connection
+            Statement stmt = conn.createStatement();
 
-			// Loop through the result set
-			while (rs.next()) {
-				long supplier,agency;
-				agency = doHash(rs.getString("agencyName"));
-				if (rs.getString("agencyName") != previousAgency) {
-					if (!inserter.nodeExists(agency)) {
-					Map<String, Object> properties = new HashMap<String, Object>();
-					  properties.put("Label", rs.getString("agencyName"));
-					  inserter.createNode(agency, properties);
-					}
-				}
-				if (rs.getString("supplierABN") != "0") {
-					supplier = doHash(rs.getString("supplierABN"));
-				} else {
-				supplier = doHash(rs.getString("supplierName"));
-				}
-				  // inject some data 
-			if (!inserter.nodeExists(supplier)) {
-				Map<String, Object> properties = new HashMap<String, Object>();
-			
-				  properties.put("Label", rs.getString("supplierName"));
-				  inserter.createNode(supplier, properties);
-			}
-				  
-				  inserter.createRelationship(agency, supplier,
-				  DynamicRelationshipType.withName("KNOWS"), null);
-				  
-			}
-			// Close the result set, statement and the connection
-			rs.close();
-			stmt.close();
-			conn.close();
-		} catch (SQLException se) {
-			System.out.println("SQL Exception:");
-
-			// Loop through the SQL Exceptions
-			while (se != null) {
-				System.out.println("State  : " + se.getSQLState());
-				System.out.println("Message: " + se.getMessage());
-				System.out.println("Error  : " + se.getErrorCode());
-
-				se = se.getNextException();
-			}
-		} catch (Exception e) {
-			System.out.println(e);
-		}
-		  // shutdown, makes sure all changes are written to disk
-		  inserter.shutdown();
-	}
-
-	static long doHash(String input) {
-		MessageDigest m;
-		try {
-			m = MessageDigest.getInstance("MD5");
-			m.reset();
-			m.update(input.getBytes());
-			byte[] digest = m.digest();
-			return new BigInteger(1, digest).longValue();
-		} catch (NoSuchAlgorithmException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		return 0;
-
-	}
+            // Execute the query
+            ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+                    + "  contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+                    + "FROM  public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
+                    + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
+                    + "  contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
+            String previousAgency = "";
+GraphDatabaseService gds = inserter.getGraphDbService(); 
+HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
+HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
+        
+            // Loop through the result set
+            while (rs.next()) {
+                long supplierID, agencyID;
+                String supplierKey;
+                    if (agencyIDs.get(rs.getString("agencyName")) == null) {
+                       Node myNode = gds.createNode(); 
+                        myNode.setProperty("Label", rs.getString("agencyName"));
+                          myNode.setProperty("type", "agency");
+                        agencyIDs.put(rs.getString("agencyName"), myNode.getId());
+                        if (myNode.getId() %100 == 0) {
+    System.out.println("Agency "+myNode.getId());
+}
+                    } 
+                       agencyID = agencyIDs.get(rs.getString("agencyName"));
+                
+                
+                if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
+                    supplierKey = rs.getString("supplierABN");
+                } else {
+                    supplierKey = rs.getString("supplierName");
+                }
+                // inject some data 
+                if (supplierIDs.get(supplierKey) == null) {
+                    Node myNode = gds.createNode(); 
+                        myNode.setProperty("Label", rs.getString("supplierName"));
+                            myNode.setProperty("type", "supplier");
+                    supplierIDs.put(supplierKey, myNode.getId());
+                    if (myNode.getId() %1000 == 0) {
+    System.out.println("Supplier "+myNode.getId());
+}
+                }
+                supplierID = supplierIDs.get(supplierKey);
 
 
+                long rel = inserter.createRelationship(agencyID, supplierID,
+                       DynamicRelationshipType.withName("KNOWS"), null);
+                inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
+
+            }
+            // Close the result set, statement and the connection
+            rs.close();
+            stmt.close();
+            conn.close();
+        } catch (SQLException se) {
+            System.out.println("SQL Exception:");
+
+            // Loop through the SQL Exceptions
+            while (se != null) {
+                System.out.println("State  : " + se.getSQLState());
+                System.out.println("Message: " + se.getMessage());
+                System.out.println("Error  : " + se.getErrorCode());
+
+                se = se.getNextException();
+            }
+        }
+//make the changes visible for reading, use this sparsely, requires IO!
+        labels.flush();
+
+// Make sure to shut down the index provider
+        indexProvider.shutdown();
+        inserter.shutdown();
+    }
+
+   
 }

--- /dev/null
+++ b/admin/partialdata/importamendments.php
@@ -1,1 +1,329 @@
-
+<?php
+include_once ("../../lib/common.inc.php");
+/*
+update contractnotice set "supplierABN" = a."supplierABN" 
+from contractnotice as cn inner join (select "supplierABN", 
+"supplierName" from contractnotice where "supplierABN" 
+IS NOT NULL and "supplierABN" != 0) as a on 
+cn."supplierName" = a."supplierName" where 
+cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN" 
+IS NULL or contractnotice."supplierABN" = 0) */
+// http://www.lastcraft.com/browser_documentation.php
+// http://code.google.com/p/phpquery/
+require('phpQuery-onefile.php');
+function getURL($url) {
+    //return file_get_contents($url);
+    $ch = curl_init($url);
+	curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+	curl_setopt($ch, CURLOPT_HEADER, 0);
+	curl_setopt($ch, CURLOPT_TIMEOUT, 45);
+curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
+curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
+	$page = curl_exec($ch);
+	if (curl_errno($ch)) {
+		echo "<font color=red> Database temporarily unavailable: ";
+		echo curl_errno($ch) . " " . curl_error($ch);
+		
+			echo $url;
+		echo "</font><br>";
+	}
+	curl_close($ch);
+	return $page;
+}
+
+
+function getTextFromTHNode($Node, $Text = "") { 
+    if ($Node->tagName == null) 
+        return $Text.$Node->textContent;
+    
+if ($Node->tagName != "td") {
+    $Node = $Node->firstChild; 
+    if ($Node != null) 
+        $Text = getTextFromTHNode($Node, $Text); 
+
+    while($Node->nextSibling != null) { 
+        $Text = getTextFromTHNode($Node->nextSibling, $Text); 
+        $Node = $Node->nextSibling; 
+    }
+}
+    return $Text; 
+} 
+
+function getTextFromNode($Node, $Text = "") { 
+    if ($Node->tagName == null) 
+        return $Text.$Node->textContent; 
+if ($Node->tagName != "th" && $Node->tagName != "span") {
+    $Node = $Node->firstChild; 
+    if ($Node != null) 
+        $Text = getTextFromNode($Node, $Text); 
+
+    while($Node->nextSibling != null) { 
+        $Text = getTextFromNode($Node->nextSibling, $Text); 
+        $Node = $Node->nextSibling; 
+    }
+}
+    return $Text; 
+} 
+function dom_to_array($root) 
+{ 
+    $result = array(); 
+
+    if ($root->hasAttributes()) 
+    { 
+        $attrs = $root->attributes; 
+
+        foreach ($attrs as $i => $attr) 
+            $result[$attr->name] = $attr->value; 
+    } 
+
+    $children = $root->childNodes; 
+if ($root->childNodes) {
+    if ($children->length == 1) 
+    { 
+        $child = $children->item(0); 
+
+        if ($child->nodeType == XML_TEXT_NODE) 
+        { 
+            $result['_value'] = $child->nodeValue; 
+
+            if (count($result) == 1) 
+                return $result['_value']; 
+            else 
+                return $result; 
+        } 
+    } 
+
+    $group = array(); 
+
+    for($i = 0; $i < $children->length; $i++) 
+    { 
+        $child = $children->item($i); 
+
+        if (!isset($result[$child->nodeName])) 
+            $result[$child->nodeName] = dom_to_array($child); 
+        else 
+        { 
+            if (!isset($group[$child->nodeName])) 
+            { 
+                $tmp = $result[$child->nodeName]; 
+                $result[$child->nodeName] = array($tmp); 
+                $group[$child->nodeName] = 1; 
+            } 
+
+            $result[$child->nodeName][] = dom_to_array($child); 
+        } 
+    } 
+}
+
+    return $result; 
+} 
+
+function importCN($cnid) {
+    global $conn;
+    $CN = str_replace("-A", "00",$cnid);
+    // check if already complete
+    $query = 'Select "parentCN" from contractnotice
+	where "CNID" = :CNID';
+	$query = $conn->prepare($query);
+	$query->bindParam(":CNID", $CN);
+	$query->execute();
+	$r = $query->fetch(PDO::FETCH_ASSOC);
+    if ($r['parentCN'] == NULL) {
+$site = "https://www.tenders.gov.au/";
+$searchResult = phpQuery::newDocument(getURL("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
+//echo "https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid;
+$url = "";
+foreach(pq('a') as $a) {
+	if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) {
+	    //echo  $a->getAttribute("href");
+	    $url = $a->getAttribute("href");
+	break;
+	}
+}
+$cn = phpQuery::newDocument(getURL($site.$url));
+	$datamapping0711 = array(
+		"Agency" => "agencyName",
+		"Parent CN" => "parentCN",
+		"CN ID" => "CNID",
+		"Publish Date" => "publishDate",
+		"Amendment Date" => "amendDate",
+		"Status" => "",
+		"StartDate" => "contractStart",
+		"EndDate" => "contractEnd",
+		"Contract Value (AUD)" => "value",
+		"Description" => "description",
+		"Agency Reference ID" => "agencyID",
+		"Category" => "category",
+		"Procurement Method" => "procurementMethod",
+		"ATM ID" => "atmID",
+		"SON ID" => "SONID",
+		"Confidentiality - Contract" => "confidentialityContract",
+		"Confidentiality Reason(s) - Contract" => "confidentialityContractReason",
+		"Confidentiality - Outputs" => "confidentialityOutputs",
+		"Confidentiality Reason(s) - Outputs" => "confidentialityOutputsReason",
+		"Consultancy" => "consultancy",
+		"Consultancy Reason(s)" => "consultancyReason",
+		"Amendment Reason" => "amendmentReason",
+		"Name" => "supplierName",
+		"Postal Address" => "supplierAddress",
+		"Town/City" => "supplierCity",
+		"Postcode" => "supplierPostcode",
+		"Country" => "supplierCountry",
+		"ABN Exempt" => "supplierABNExempt",
+		"ABN" => "supplierABN",
+		"Branch" => "contactBranch",
+		"Division" => "contactDivision",
+		"Office Postcode" => "contactPostcode"
+	);
+$cnFields = Array();
+foreach(pq('tr') as $tr) {
+	$tra = dom_to_array($tr);
+	if (is_array($tra['th'])) {
+	    $fieldName = trim(getTextFromTHNode($tr));
+	} else {
+	$fieldName = trim(str_replace("/th>","",$tra['th']));
+	}
+	$fieldValue = trim(print_r($tra['td'],true));
+	if ($fieldName == "State/Territory" || $fieldName == "Contact Name"
+	    || $fieldName == "Contact Phone" || $fieldName == "Contact Email"
+	    ||$fieldName == "Amendments") {
+	    // do nothing
+	} else if ($fieldName == "Contract Period") {
+	    $contractPeriod = explode("to",$fieldValue);
+	    $cnFields["contractStart"] =  trim($contractPeriod[0]);
+		$cnFields["contractEnd"] =  trim($contractPeriod[1]);
+	} else {
+	    $fieldName = $datamapping0711[$fieldName];
+
+	if ($fieldName == "parentCN" || $fieldName == "CNID") {
+	    	    if (is_array($tra['td'])) {
+							    $fieldValue = trim(getTextFromNode($tr));
+							    }
+							$fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
+							$fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
+	} elseif ($fieldName == "description") {
+	   
+							if (is_array($tra['td'])) $fieldValue = print_r($tra['td']['p'],true);
+							
+	} elseif ($fieldName == "value" || $fieldName == "supplierABN") {
+							if (is_array($tra['td'])) {
+							    $fieldValue = trim(getTextFromNode($tr));
+							    }
+							$fieldValue = str_replace(Array("$",","," "), "", $fieldValue);
+							//if (!is_numeric($fieldValue)) $fieldValue = 0;
+							if ($fieldValue == "Exempt") $fieldValue = NULL;
+	} elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
+							$fieldValue = date('Y-m-d H:i:s', strtotime($fieldValue));
+						} elseif (is_array($tra['td'])) {
+							    $fieldValue = trim(getTextFromNode($tr));
+							    }
+	    echo $fieldName. " = " .$fieldValue."<br>\n";
+	    $cnFields[$fieldName] = $fieldValue;
+	}
+}
+if (isset($cnFields[""])) {
+ $cnFields["description"] .= $cnFields[""];
+ unset($cnFields[""]);
+}
+$cnFields["importFile"] = $url;
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
+		for($key = 0; $key < sizeof($cnFields); $key++) {
+			$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+		}
+		$contractNoticeInsertQ.= ");";
+		//echo $contractNoticeInsertQ;
+		$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+		$contractNoticeInsertQ->execute(array_values($cnFields));
+				$errors = $conn->errorInfo();
+				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+					echo "Dupe {$cnFields['CNID']}<br>";
+				}
+				elseif ($errors[1] == 0) {
+					echo "Success insert {$cnFields['CNID']} <br>";
+				}
+				else {
+					foreach ($cnFields as $key => $cnf) {
+						echo var_dump($key) . $cnf . "<br>";
+					}
+					echo $cnFields['CNID'] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+				}
+$contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
+$count = 0;
+		foreach ($cnFields as $key => $f) {
+		    
+			$count++;
+			$contractNoticeUpdateQ.= '"'.$key.'"=? '.($count >= sizeof($cnFields) ? "" : ", ");
+		}
+		$contractNoticeUpdateQ.= ' WHERE "CNID"=?;';
+		$cnFields[] = $cnFields["CNID"];
+		//echo $contractNoticeUpdateQ;
+		$contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
+		$contractNoticeUpdateQ->execute(array_values($cnFields));
+				$errors = $conn->errorInfo();
+				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+				    print_r($errors);
+					echo "Dupe update {$cnFields['CNID']}<br>";
+				}
+				elseif ($errors[1] == 0) {
+					echo "Success update {$cnFields['CNID']} <br>";
+				}
+				else {
+					foreach ($cnFields as $key => $cnf) {
+						echo var_dump($key) . $cnf . "<br>";
+					}
+					echo $cnFields['CNID'] . " failed CN update.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+				}
+		}
+}
+function processFile($fpath, $tablename)
+{
+	global $conn;
+	echo " ============== $fpath  ============== <br>";
+	
+	$handle = fopen($fpath, "r");
+	flush();
+	$row = 1;
+	
+	while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+		if ($row > 3) {
+				$data[0] = trim($data[0], "=");
+				$data[0] = trim($data[0], "\"");
+				if (strpos($data[0], "-A") > 0) {
+				    echo "Loading {$data[0]} ... <br>\n";
+				    importCN(str_replace("CN","",$data[0]));
+				}
+			}
+			flush();
+			//echo "<hr>\n";
+			
+	
+		$row++;
+	}
+	fclose($handle);
+}
+$path = './';
+if ($_REQUEST["fname"] == "") {
+	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+	$dhandle = opendir($path);
+	// define an array to hold the files
+	$files = array();
+	if ($dhandle) {
+		// loop through all of the files
+		while (false !== ($fname = readdir($dhandle))) {
+			if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) {
+				echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
+				processFile($path . $fname, "contractnotice");
+			}
+		}
+	}
+}
+else {
+	$success = 0;
+	$fname = $_REQUEST["fname"];
+	
+	$success+= processFile($path . $fname, "contractnotice");
+
+}
+
+?>
+

--- a/admin/partialdata/scrapesingle.php
+++ /dev/null
@@ -1,881 +1,1 @@
-<?php
-include_once ("../../lib/common.inc.php");
 
-$cnid = 1234;
-// http://www.lastcraft.com/browser_documentation.php
-// http://code.google.com/p/phpquery/
-require('phpQuery-onefile.php');
-function dom_to_array($root) 
-{ 
-    $result = array(); 
-
-    if ($root->hasAttributes()) 
-    { 
-        $attrs = $root->attributes; 
-
-        foreach ($attrs as $i => $attr) 
-            $result[$attr->name] = $attr->value; 
-    } 
-
-    $children = $root->childNodes; 
-if ($root->childNodes) {
-    if ($children->length == 1) 
-    { 
-        $child = $children->item(0); 
-
-        if ($child->nodeType == XML_TEXT_NODE) 
-        { 
-            $result['_value'] = $child->nodeValue; 
-
-            if (count($result) == 1) 
-                return $result['_value']; 
-            else 
-                return $result; 
-        } 
-    } 
-
-    $group = array(); 
-
-    for($i = 0; $i < $children->length; $i++) 
-    { 
-        $child = $children->item($i); 
-
-        if (!isset($result[$child->nodeName])) 
-            $result[$child->nodeName] = dom_to_array($child); 
-        else 
-        { 
-            if (!isset($group[$child->nodeName])) 
-            { 
-                $tmp = $result[$child->nodeName]; 
-                $result[$child->nodeName] = array($tmp); 
-                $group[$child->nodeName] = 1; 
-            } 
-
-            $result[$child->nodeName][] = dom_to_array($child); 
-        } 
-    } 
-}
-
-    return $result; 
-} 
-
-$site = "https://www.tenders.gov.au/";
-//$cn = phpQuery::newDocument(file_get_contents("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
-$searchResult = phpQuery::newDocument('<table class="four-col">
-		<tbody><tr>
-			<th>CN ID</th>
-			
-			<td><a href="/?event=public.cn.view&amp;CNUUID=BB3B13EA-A700-39A5-F003088ACF798AF9">CN1234</a></td>
-			
-				<th>
-				</th><td>
-			
-		</td></tr>
-		<tr>
-			<th>Agency</th>
-			<td colspan="3">Department of Veterans Affairs</td>
-		</tr>
-		<tr>
-			<th>Publish Date</th>
-			<td colspan="3"> 18-Aug-2006 </td>
-		</tr>
-	
-		<tr>
-			<th>Category</th>
-			<td colspan="3">Computer services</td>
-		</tr>
-		
-		<tr>
-			<th>Contract Period</th>
-			<td colspan="3"> 1-Aug-2006  to 31-Jul-2007 </td>
-		</tr>
-		<tr>
-			<th>Contract Value (AUD)</th>
-			<td colspan="3"> $156,200.00 </td>
-		</tr>
-		
-		<tr>
-			<th>Supplier Name</th>
-			<td colspan="3">WIZARD INFORMATION SERVICES PTY LTD</td>
-		</tr>
-		<tr>
-			<th></th>
-			<td colspan="3">
-				<p>
-					<span class="last-updated"><strong>Last Updated:</strong> 18-Aug-2006 4:51 pm  (ACT Local time)</span>
-					
-					<strong><a href="/?event=public.cn.view&amp;CNUUID=BB3B13EA-A700-39A5-F003088ACF798AF9">Full Details</a></strong>
-					
-				</p>
-			</td>
-		</tr>
-	</tbody></table>');
-foreach(pq('a') as $a) {
-	if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) echo  $a->getAttribute("href");
-	break;
-}
-$cn = phpQuery::newDocument('
-
-
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-
-<html lang="en-AU">
-
-<head>
-
-	<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-
-	<title>AusTender: Contract Notice View - CN1234</title>
-
-	
-
-	<meta name="language" content="en-AU">
-
-	<meta name="description" content="AusTender provides centralised publication of Australian Government business opportunities, annual procurement plans, multi-use lists and contracts awarded.">
-
-	<meta name="keywords" content="tenders, australian government tenders, austender, austenders, australian tenders, federal government tenders">
-
-
-
-	<link rel="stylesheet" type="text/css" href="/styles/styles_frontend_main.css" media="all">
-
-	<link rel="stylesheet" type="text/css" href="/styles/styles_frontend_print.css" media="print">
-
-	<link rel="stylesheet" type="text/css" href="/styles/styles_reports.css" media="all">
-
-	<link rel="stylesheet" href="/styles/calendar-atii.css" type="text/css" media="all">
-
-	<link rel="shortcut icon" href="/favicon.ico">
-
-	
-
-	
-
-</head>
-
-
-
-<body>
-
-
-
-<div id="header">
-
-	<a href="/"><img src="/images/header_logo.gif" alt="AusTender - The Australian Government Tender System"></a>
-
-</div>
-
-
-
-<form action="./" method="get" id="header-tools" onSubmit="return (this.keyword.value.length != 0)">
-
-	
-
-	<input
-
-		name="event"
-
-		id="form-event"
-
-		type="hidden"
-
-		value="public.advancedsearch.keyword"
-
-		>
-
-
-
-	<a href="/?event=public.home" id="home-link">Home</a>
-
-	<label class="hidden" for="search-text">Search</label>
-
-	<input type="text" value="" name="keyword" id="search-text">
-
-	<input type="submit" value="Search" id="search-submit">
-
-	<a href="?event=public.advancedsearch.home">Advanced Search</a>
-
-</form>
-
-
-
-<div id="left-col-wrapper">
-
-<div id="left-col"><div class="pad">
-
-	
-
-<form action="?event=public.login" method="post" enctype="multipart/form-data" id="login-form">
-
-	<label for="login-username">Username</label> (email)
-
-	<br>
-
-	<input type="text" name="pub-auth-username" id="login-username" value="">
-
-
-
-	<label for="login-password">Password</label>
-
-	<br>
-
-	<input type="password" name="pub-auth-password" id="login-password" value="">
-
-	<input type="Submit" value="Login" id="login-submit">
-
-
-
-	<p>
-
-		&raquo; <a href="?event=public.registereduser.forgotPassFormStep1">Forgotten password?</a>
-
-		<br>
-
-		&raquo; <a href="?event=public.registereduser.new">New user registration</a>
-
-	</p>
-
-
-
-</form> 
-
-			<h2>View</h2>
-
-		
-
-			<ul>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.atmproposed.list" title="Information about potential procurements prior to their release to the market">Pre-Release Notices</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.ATM.list" title="Business opportunities that are currently out to the market">Current ATMs</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.ATM.closed" title="Business opportunities that have closed in the last 30 days">Closed ATMs</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.CN.search" title="Notices of contracts awarded to suppliers">Contract Notices</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.SON.search" title="Notices of standing arrangements with suppliers, including panels and period contracts">Standing Offer Notices</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.MUL.list" title="Multi-Use Lists (MULs) currently open for applications for inclusion">Current Multi-Use Lists</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.MUL.closed" title="Multi-Use Lists (MULs) that have closed in the last 30 days">Closed Multi-Use Lists</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.APP.list" title="Agencies Planned Procurements for the latest financial year">Procurement Plans</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.reports.list" title="A list of standard reports on various types of procurement information">Reports</a>
-
-				</li>
-
-			
-
-			</ul>
-
-		
-
-			<h2>Info &amp; Links</h2>
-
-		
-
-			<ul>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.agency.list" title="A Link to Australian Government agency and department corporate addresses">Agency Addresses</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.relatedlink.list" title="Links to other procurement related websites, national and international">Related Links</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.help.list" title="A list of frequently asked questions and a link to download the AusTender Public User Guide">Help</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.contactus.show" title="Contact information for the AusTender Help Desk">Contact Us</a>
-
-				</li>
-
-			
-
-			</ul>
-
-		
-
-			<h2>Policies</h2>
-
-		
-
-			<ul>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.policydocs.list" title="Links to Australian Government procurement policy information">Policy Documents</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.document.list" title="Australian Government standard form contracts, templates and procurement related guidance">Procurement Document Library</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.termsOfUse" title="Conditions of use for both the AusTender website and ATM document distribution">Terms of Use</a>
-
-				</li>
-
-			
-
-					<li>
-
-				
-
-					<a href="?event=public.privacyStatement" title="The terms under which we manage information relating to and provided by AusTender Registered Users">Privacy Statement</a>
-
-				</li>
-
-			
-
-			</ul>
-
-		
-
-</div></div>
-
-
-
-</div>
-
-
-
-
-
-
-
-
-
-<div id="main-content"><div class="pad">
-
-	
-
-<ol id="bread-crumbs">
-
-	
-
-			<li class="first-item">
-
-				<a href="./?event=public.home">Home</a>
-
-			</li>
-
-		
-
-	<li> Contract Notice View - CN1234</li>
-
-</ol>
-
-
-
-
-
-	<h1> Contract Notice View - CN1234</h1>
-
-
-
-	
-
-		<div id="container">
-
-        
-
-
-
-<div class="content">
-
-
-
-	<div id="intro">
-
-		<span><p><span>AusTender holds Contract and Standing Offer Notices for the 07/08 financial year forward. For information related to previous years, please contact the AusTender Help Desk.</span></p><span><p><strong><span>Subcontractors:&nbsp; For Commonwealth contracts that started on or after 1 December 2008, agencies are required to provide the names of any associated subcontractors on request.&nbsp; Information&nbsp;on subcontractors&nbsp;can be&nbsp;sought&nbsp;</span><span>directly from the relevant agency through the&nbsp;Agency Contact listed in each Contract Notice.</span></strong></p></span></span> 
-
-	</div>
-
-
-
-</div>
-
-
-
-		</div>
-
-	<h2 class="highlight">IMU Contract Programmer: IMU-ICT040 (GAPS ID: 1611946)</h2>
-
-
-
-<table>
-
-	<tr>
-
-		<th>CN ID</th>
-
-		
-
-		<td>CN1234</td>
-
-		
-
-	</tr>
-
-	<tr>
-
-		<th>Agency</th>
-
-		<td>Department of Veterans Affairs</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Publish Date</th>
-
-		<td> 18-Aug-2006 </td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Category</th>
-
-		<td>Computer services</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Contract Period</th>
-
-		<td>
-
-			1-Aug-2006  to 31-Jul-2007 
-
-		</td>
-
-	</tr>	
-
-		<tr>
-
-			<th>Contract Value (AUD)</th>
-
-		<td>
-
-			$156,200.00 
-
-		</td>
-
-		</tr>
-
-	
-
-	<tr>
-
-		<th>Description</th>
-
-		<td>IMU Contract Programmer: IMU-ICT040 (GAPS ID: 1611946)</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Procurement Method</th>
-
-		<td>Open</td>
-
-	</tr>
-
-	
-
-
-
-	<tr>
-
-		<th>Confidentiality - Contract</th>
-
-		<td>
-
-			
-
-		</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Confidentiality - Outputs</th>
-
-		<td>
-
-			No 
-
-		</td>
-
-	</tr>
-
-	
-
-			<tr>
-
-				<th>Consultancy</th>
-
-				<td>No</td>
-
-			</tr>
-
-		
-
-
-
-</table>
-
-
-
-<h2>Supplier Details</h2>
-
-<table>
-
-	<tr>
-
-		<th>Name</th>
-
-		<td>
-
-			WIZARD INFORMATION SERVICES PTY LTD 
-
-		</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Postal Address</th>
-
-		<td>GPO Box 2700</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Town/City</th>
-
-		<td>CANBERRA CITY</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Postcode</th>
-
-		<td>2601</td>
-
-	</tr>
-
-	<tr>
-
-		<th>State/Territory</th>
-
-		<td>ACT</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Country</th>
-
-		<td>Australia</td>
-
-	</tr>
-
-	<tr>
-
-		<th>ABN</th>		
-
-		<td>
-
-			47 008 617 561 
-
-		</td>
-
-		
-
-	</tr>
-
-</table>
-
-
-
-
-
-
-
-<h2>Agency Details</h2>
-
-
-
-<table class="medium-th">
-
-	<tr>
-
-		<th>Contact Name/th>
-
-		<td>Monico, Raymond J</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Contact Phone</th>
-
-		<td>(02) 6289 6016</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Branch</th>
-
-		<td>Information Management Unit</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Division</th>
-
-		<td>National Office Compensation Division</td>
-
-	</tr>
-
-	
-
-	<tr>
-
-		<th>Office Postcode</th>
-
-		<td>2606</td>
-
-	</tr>
-
-	<tr>
-
-		<th>Agency Reference ID</th>
-
-		<td>107460</td>
-
-	</tr>
-
-</table> </div></div>
-
-
-
-
-
-
-
-</body>
-
-</html> ');
-	$datamapping0711 = array(
-		"Agency" => "agencyName",
-		"Parent CN ID" => "parentCN",
-		"CN ID" => "CNID",
-		"Publish Date" => "publishDate",
-		"Amendment Date" => "amendDate",
-		"Status" => "",
-		"StartDate" => "contractStart",
-		"EndDate" => "contractEnd",
-		"Contract Value (AUD)" => "value",
-		"Description" => "description",
-		"Agency Reference ID" => "agencyID",
-		"Category" => "category",
-		"Procurement Method" => "procurementMethod",
-		"ATM ID" => "atmID",
-		"SON ID" => "SONID",
-		"Confidentiality - Contract" => "confidentialityContract",
-		"Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
-		"Confidentiality - Outputs" => "confidentialityOutputs",
-		"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
-		"Consultancy" => "consultancy",
-		"Consultancy Reason(s)" => "consultancyReason",
-		"Amendment Reason" => "amendmentReason",
-		"Name" => "supplierName",
-		"Postal Address" => "supplierAddress",
-		"Town/City" => "supplierCity",
-		"Postcode" => "supplierPostcode",
-		"Country" => "supplierCountry",
-		"ABN Exempt" => "supplierABNExempt",
-		"ABN" => "supplierABN",
-		"Branch" => "contactBranch",
-		"Division" => "contactDivision",
-		"Office Postcode" => "contactPostcode"
-	);
-$cnFields = Array();
-foreach(pq('tr') as $tr) {
-	$tra = dom_to_array($tr);
-	$fieldName = trim(str_replace("/th>","",$tra['th']));
-	$fieldValue = trim(print_r($tra['td'],true));
-	if ($fieldName == "State/Territory" || $fieldName == "Contact Name" || $fieldName == "Contact Phone") {
-	    // do nothing
-	} else if ($fieldName == "Contract Period") {
-	    $contractPeriod = explode("to",$fieldValue);
-	    $cnFields["contractStart"] =  trim($contractPeriod[0]);
-		$cnFields["contractEnd"] =  trim($contractPeriod[1]);
-	} else {
-	    $fieldName = $datamapping0711[$fieldName];
-	if ($fieldName == "parentCN" || $fieldName == "CNID") {
-							$fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
-							$fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
-	} elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
-							$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($fieldValue));
-						}
-echo $fieldName. " = " .$fieldValue."<br>\n";
-$cnFields[$fieldName] = $fieldValue;
-	}
-}
-$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
-		for($key = 0; $key < sizeof($cnFields); $key++) {
-			$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
-		}
-		$contractNoticeInsertQ.= ");";
-		echo $contractNoticeInsertQ;
-		//$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
-		
-$contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
-$count = 0;
-		foreach ($cnFields as $key => $f) {
-		    
-			$count++;
-			$contractNoticeUpdateQ.= $key."=? ".($count >= sizeof($cnFields) ? "" : ", ");
-		}
-		$contractNoticeUpdateQ.= " WHERE CNID=?;";
-		$cnFields[] = $cnFields["CNID"];
-		echo $contractNoticeUpdateQ;
-		$contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
-?>

--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -10,12 +10,16 @@
 	// some Australian spellings
 	$isiz = str_replace("iz","is",$row['Title']);
 	$unspsc[$isiz] = $row['UNSPSC'];
+	$filfill = str_replace("fill","fil",$row['Title']);
+	$unspsc[$filfill] = $row['UNSPSC'];
 	$defence = str_replace("efense","efence",$row['Title']);
 	$unspsc[$defence] = $row['UNSPSC'];
 		$armor = str_replace("rmored","rmoured",$row['Title']);
 	$unspsc[$armor] = $row['UNSPSC'];
-	$center = str_replace("enter","entre",$row['Title']);
-	$unspsc[$center] = $row['UNSPSC'];
+	$erre = str_replace("er","re",$row['Title']);
+	$unspsc[$erre] = $row['UNSPSC'];
+	$lyslyz = str_replace("lyz","lys",$row['Title']);
+	$unspsc[$lyslyz] = $row['UNSPSC'];
 	// some divergence from standard
 	$tobacco = str_replace("Food Beverage and Tobacco Products","Food and Beverage Products",$row['Title']);
 	$unspsc[$tobacco] = $row['UNSPSC'];

--- a/exportData.csv.php
+++ b/exportData.csv.php
@@ -5,12 +5,14 @@
 // source: http://stackoverflow.com/questions/81934/easy-way-to-export-a-sql-table-without-access-to-the-server-or-phpmyadmin#81951
 $query = $conn->prepare('
 SELECT "CNID",contractnotice."agencyName",agency.abn as "agencyABN",EXTRACT(EPOCH FROM "publishDate") as "publishDate",EXTRACT(EPOCH FROM "contractStart") as "contractStart",EXTRACT(EPOCH FROM "contractEnd") as "contractEnd",value,description,"procurementMethod",category,"categoryUNSPSC", 
- "supplierABN","supplierName","supplierCity","supplierPostcode","supplierCountry","contactPostcode",
+ "supplierABN","supplierName",
 (
  case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as supplierID,
  
 (\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as sourceURL 
-FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0');
+FROM contractnotice join agency on contractnotice."agencyName"=agency."agencyName" where "childCN" = 0'
+, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
+// "supplierCity","supplierPostcode","supplierCountry","contactPostcode",
 // (substr( "categoryUNSPSC"::text, 0, 2 ) || \'0000000\'::text) as "categoryUNSPSClv1", "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 3 ) || \'000000\'::text) as "categoryUNSPSClv2" "categoryUNSPSC", (substr( "categoryUNSPSC"::text, 0, 4 ) || \'00000\'::text as "categoryUNSPSClv3")
 $query->execute();
 if (!$query) {
@@ -31,7 +33,7 @@
     header('Pragma: no-cache');
     header('Expires: 0');
     fputcsv($fp, $headers);
-foreach ($query->fetchAll(PDO::FETCH_NUM) as $row) {
+ while ($row = $query->fetch(PDO::FETCH_NUM, PDO::FETCH_ORI_NEXT)) {
 	foreach ($row as $key => &$colvalue) {
        
 			$colvalue =  preg_replace( '/[^[:print:]]/', '',

--- a/lib/graphs.inc.php
+++ b/lib/graphs.inc.php
@@ -7,6 +7,7 @@
     <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.js"></script> 
         <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.pie.js"></script> 
               <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.selection.js"></script> 
+              <script language="javascript" type="text/javascript" src="lib/flot/jquery.flot.stack.js"></script> 
 
 ';
 $includedFlot = true;
@@ -23,8 +24,9 @@
 	 $(function () {
 
  var d1 = [];
+ var d2 = [];
 <?php
-$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999) as a group by cnid order by cnid';
+$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is null) as a group by cnid order by cnid';
 $query = $conn->prepare($query);
 $query->execute();
 if (!$query) {
@@ -35,6 +37,18 @@
 foreach ($query->fetchAll() as $delta) {
 
 	echo "d1.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
+};
+$query = 'select cnid, count(*) from (select ("CNID" - MOD("CNID",100)) as cnid from contractnotice where "CNID" < 999999 and "parentCN" is not null) as a group by cnid order by cnid';
+$query = $conn->prepare($query);
+$query->execute();
+if (!$query) {
+	databaseError($conn->errorInfo());
+	return Array();
+}
+
+foreach ($query->fetchAll() as $delta) {
+
+	echo "d2.push([ ".intval($delta['cnid']).", ".intval($delta['count'])."]); \n";
 };
 ?>
 
@@ -47,10 +61,20 @@
                },
             bars: { show: true }
         },
+       {
+            data: d2,
+series: {
+                   lines: { show: true },
+                   points: { show: true }
+               },
+            bars: { show: true }
+        },
     ];
     var options = 
     {
-
+series: {
+                stack: true,
+            },
             grid: { hoverable: true, clickable: true, labelMargin: 17  },
             selection: { mode: "x" }
     };