Merge branch 'master' of ssh://apples.lambdacomplex.org/git/contractdashboard
--- a/admin/import.php
+++ b/admin/import.php
@@ -261,10 +261,10 @@
}
// run post import data processing
// cn
-//include ("linkAmendments.php");
+include ("linkAmendments.php");
include ("updateUNSPSC.php");
// agency
-include ("setAgencyStatus.php");
-include ("setAgencyURLABN.php");
+//include ("setAgencyStatus.php");
+//include ("setAgencyURLABN.php");
?>
--- /dev/null
+++ b/admin/neo4jimporter/nbactions.xml
@@ -1,1 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+ <action>
+ <actionName>run</actionName>
+ <goals>
+ <goal>process-classes</goal>
+ <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+ </goals>
+ <properties>
+ <exec.classpathScope>runtime</exec.classpathScope>
+ <exec.args>-Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+ <exec.executable>java</exec.executable>
+ </properties>
+ </action>
+ <action>
+ <actionName>debug</actionName>
+ <goals>
+ <goal>process-classes</goal>
+ <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+ </goals>
+ <properties>
+ <exec.classpathScope>runtime</exec.classpathScope>
+ <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+ <jpda.listen>true</jpda.listen>
+ <exec.executable>java</exec.executable>
+ </properties>
+ </action>
+ <action>
+ <actionName>profile</actionName>
+ <goals>
+ <goal>process-classes</goal>
+ <goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
+ </goals>
+ <properties>
+ <exec.args>${profiler.args} -Xmx4048M -server -classpath %classpath ${packageClassName}</exec.args>
+ <profiler.action>profile</profiler.action>
+ <exec.executable>${profiler.java}</exec.executable>
+ </properties>
+ </action>
+ <action>
+ <actionName>run</actionName>
+ <goals>
+ <goal>process-classes</goal>
+ <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+ </goals>
+ <properties>
+ <exec.classpathScope>runtime</exec.classpathScope>
+ <exec.args>-classpath %classpath Importer</exec.args>
+ <exec.executable>java</exec.executable>
+ </properties>
+ </action>
+ </actions>
--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -6,8 +6,8 @@
<dependencies>
<dependency>
<groupId>org.neo4j</groupId>
- <artifactId>neo4j-kernel</artifactId>
- <version>1.4</version>
+ <artifactId>neo4j</artifactId>
+ <version>1.5</version>
</dependency>
<dependency>
<groupId>postgresql</groupId>
--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -1,3 +1,4 @@
+
import java.io.ObjectInputStream.GetField;
import java.math.BigInteger;
import java.security.MessageDigest;
@@ -12,144 +13,147 @@
import java.util.Map;
import org.neo4j.graphdb.DynamicRelationshipType;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.Node;
+import org.neo4j.graphdb.index.BatchInserterIndex;
+import org.neo4j.graphdb.index.BatchInserterIndexProvider;
+import org.neo4j.helpers.collection.MapUtil;
import org.neo4j.kernel.impl.batchinsert.BatchInserter;
import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
+import org.neo4j.index.impl.lucene.*;
public class Importer {
- public static void main(String[] argv) {
-
- Map<String,String> props = new HashMap<String, String>();
- props.put("neostore.nodestore.db.mapped_memory", "22000000"); // <expected number of nodes * 9 bytes>
- props.put("neostore.relationshipstore.db.mapped_memory", "22000000"); // <expected number of relationships * 33 bytes>
- // create the batch inserter
- BatchInserter inserter = new
- BatchInserterImpl("neo4j-db/",props
- );
-
+ public static void main(String[] argv) {
+ BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
+ BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
+ BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
+ labels.setCacheCapacity("Label", 100000);
-
-
- System.out.println("-------- PostgreSQL "
- + "JDBC Connection Testing ------------");
- try {
+ System.out.println("-------- PostgreSQL "
+ + "JDBC Connection Testing ------------");
- Class.forName("org.postgresql.Driver");
+ try {
- } catch (ClassNotFoundException e) {
+ Class.forName("org.postgresql.Driver");
- System.out.println("Where is your PostgreSQL JDBC Driver? "
- + "Include in your library path!");
- e.printStackTrace();
+ } catch (ClassNotFoundException e) {
- }
+ System.out.println("Where is your PostgreSQL JDBC Driver? "
+ + "Include in your library path!");
+ e.printStackTrace();
- System.out.println("PostgreSQL JDBC Driver Registered!");
+ }
- Connection conn = null;
+ System.out.println("PostgreSQL JDBC Driver Registered!");
- try {
+ Connection conn = null;
- conn = DriverManager.getConnection(
- "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
- "postgres", "snmc");
+ try {
- } catch (SQLException e) {
+ conn = DriverManager.getConnection(
+ "jdbc:postgresql://127.0.0.1:5432/contractDashboard",
+ "postgres", "snmc");
- System.out.println("Connection Failed! Check output console");
- e.printStackTrace();
+ } catch (SQLException e) {
- }
+ System.out.println("Connection Failed! Check output console");
+ e.printStackTrace();
- if (conn != null) {
- System.out.println("You made it, take control your database now!");
- } else {
- System.out.println("Failed to make connection!");
- }
- try {
- // Print all warnings
- for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn
- .getNextWarning()) {
- System.out.println("SQL Warning:");
- System.out.println("State : " + warn.getSQLState());
- System.out.println("Message: " + warn.getMessage());
- System.out.println("Error : " + warn.getErrorCode());
- }
+ }
- // Get a statement from the connection
- Statement stmt = conn.createStatement();
+ if (conn != null) {
+ System.out.println("You made it, take control your database now!");
+ } else {
+ System.out.println("Failed to make connection!");
+ }
+ try {
+ // Print all warnings
+ for (SQLWarning warn = conn.getWarnings(); warn != null; warn = warn.getNextWarning()) {
+ System.out.println("SQL Warning:");
+ System.out.println("State : " + warn.getSQLState());
+ System.out.println("Message: " + warn.getMessage());
+ System.out.println("Error : " + warn.getErrorCode());
+ }
- // Execute the query
- ResultSet rs = stmt.executeQuery("SELECT distinct contractnotice.\"agencyName\", contractnotice.\"supplierABN\", contractnotice.\"supplierName\" FROM public.contractnotice limit 300;");
-String previousAgency = "";
+ // Get a statement from the connection
+ Statement stmt = conn.createStatement();
- // Loop through the result set
- while (rs.next()) {
- long supplier,agency;
- agency = doHash(rs.getString("agencyName"));
- if (rs.getString("agencyName") != previousAgency) {
- if (!inserter.nodeExists(agency)) {
- Map<String, Object> properties = new HashMap<String, Object>();
- properties.put("Label", rs.getString("agencyName"));
- inserter.createNode(agency, properties);
- }
- }
- if (rs.getString("supplierABN") != "0") {
- supplier = doHash(rs.getString("supplierABN"));
- } else {
- supplier = doHash(rs.getString("supplierName"));
- }
- // inject some data
- if (!inserter.nodeExists(supplier)) {
- Map<String, Object> properties = new HashMap<String, Object>();
-
- properties.put("Label", rs.getString("supplierName"));
- inserter.createNode(supplier, properties);
- }
-
- inserter.createRelationship(agency, supplier,
- DynamicRelationshipType.withName("KNOWS"), null);
-
- }
- // Close the result set, statement and the connection
- rs.close();
- stmt.close();
- conn.close();
- } catch (SQLException se) {
- System.out.println("SQL Exception:");
-
- // Loop through the SQL Exceptions
- while (se != null) {
- System.out.println("State : " + se.getSQLState());
- System.out.println("Message: " + se.getMessage());
- System.out.println("Error : " + se.getErrorCode());
-
- se = se.getNextException();
- }
- } catch (Exception e) {
- System.out.println(e);
- }
- // shutdown, makes sure all changes are written to disk
- inserter.shutdown();
- }
-
- static long doHash(String input) {
- MessageDigest m;
- try {
- m = MessageDigest.getInstance("MD5");
- m.reset();
- m.update(input.getBytes());
- byte[] digest = m.digest();
- return new BigInteger(1, digest).longValue();
- } catch (NoSuchAlgorithmException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return 0;
-
- }
+ // Execute the query
+ ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
+ + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ + "FROM public.contractnotice where contractnotice.\"agencyName\" != 'Department of Defence'"
+ + " AND contractnotice.\"agencyName\" != 'Defence Materiel Organisation' GROUP BY contractnotice.\"agencyName\", "
+ + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
+ String previousAgency = "";
+GraphDatabaseService gds = inserter.getGraphDbService();
+HashMap<String,Long> supplierIDs = new HashMap<String,Long>();
+HashMap<String,Long> agencyIDs = new HashMap<String,Long>();
+
+ // Loop through the result set
+ while (rs.next()) {
+ long supplierID, agencyID;
+ String supplierKey;
+ if (agencyIDs.get(rs.getString("agencyName")) == null) {
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("agencyName"));
+ myNode.setProperty("type", "agency");
+ agencyIDs.put(rs.getString("agencyName"), myNode.getId());
+ if (myNode.getId() %100 == 0) {
+ System.out.println("Agency "+myNode.getId());
+}
+ }
+ agencyID = agencyIDs.get(rs.getString("agencyName"));
+
+
+ if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
+ supplierKey = rs.getString("supplierABN");
+ } else {
+ supplierKey = rs.getString("supplierName");
+ }
+ // inject some data
+ if (supplierIDs.get(supplierKey) == null) {
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("supplierName"));
+ myNode.setProperty("type", "supplier");
+ supplierIDs.put(supplierKey, myNode.getId());
+ if (myNode.getId() %1000 == 0) {
+ System.out.println("Supplier "+myNode.getId());
+}
+ }
+ supplierID = supplierIDs.get(supplierKey);
+ long rel = inserter.createRelationship(agencyID, supplierID,
+ DynamicRelationshipType.withName("KNOWS"), null);
+ inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
+
+ }
+ // Close the result set, statement and the connection
+ rs.close();
+ stmt.close();
+ conn.close();
+ } catch (SQLException se) {
+ System.out.println("SQL Exception:");
+
+ // Loop through the SQL Exceptions
+ while (se != null) {
+ System.out.println("State : " + se.getSQLState());
+ System.out.println("Message: " + se.getMessage());
+ System.out.println("Error : " + se.getErrorCode());
+
+ se = se.getNextException();
+ }
+ }
+//make the changes visible for reading, use this sparsely, requires IO!
+ labels.flush();
+
+// Make sure to shut down the index provider
+ indexProvider.shutdown();
+ inserter.shutdown();
+ }
+
+
}
Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
--- /dev/null
+++ b/admin/partialdata/importamendments.php
@@ -1,1 +1,329 @@
-
+<?php
+include_once ("../../lib/common.inc.php");
+/*
+update contractnotice set "supplierABN" = a."supplierABN"
+from contractnotice as cn inner join (select "supplierABN",
+"supplierName" from contractnotice where "supplierABN"
+IS NOT NULL and "supplierABN" != 0) as a on
+cn."supplierName" = a."supplierName" where
+cn."CNID"=contractnotice."CNID" and (contractnotice."supplierABN"
+IS NULL or contractnotice."supplierABN" = 0) */
+// http://www.lastcraft.com/browser_documentation.php
+// http://code.google.com/p/phpquery/
+require('phpQuery-onefile.php');
+function getURL($url) {
+ //return file_get_contents($url);
+ $ch = curl_init($url);
+ curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+ curl_setopt($ch, CURLOPT_HEADER, 0);
+ curl_setopt($ch, CURLOPT_TIMEOUT, 45);
+curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
+curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
+ $page = curl_exec($ch);
+ if (curl_errno($ch)) {
+ echo "<font color=red> Database temporarily unavailable: ";
+ echo curl_errno($ch) . " " . curl_error($ch);
+
+ echo $url;
+ echo "</font><br>";
+ }
+ curl_close($ch);
+ return $page;
+}
+
+
+function getTextFromTHNode($Node, $Text = "") {
+ if ($Node->tagName == null)
+ return $Text.$Node->textContent;
+
+if ($Node->tagName != "td") {
+ $Node = $Node->firstChild;
+ if ($Node != null)
+ $Text = getTextFromTHNode($Node, $Text);
+
+ while($Node->nextSibling != null) {
+ $Text = getTextFromTHNode($Node->nextSibling, $Text);
+ $Node = $Node->nextSibling;
+ }
+}
+ return $Text;
+}
+
+function getTextFromNode($Node, $Text = "") {
+ if ($Node->tagName == null)
+ return $Text.$Node->textContent;
+if ($Node->tagName != "th" && $Node->tagName != "span") {
+ $Node = $Node->firstChild;
+ if ($Node != null)
+ $Text = getTextFromNode($Node, $Text);
+
+ while($Node->nextSibling != null) {
+ $Text = getTextFromNode($Node->nextSibling, $Text);
+ $Node = $Node->nextSibling;
+ }
+}
+ return $Text;
+}
+function dom_to_array($root)
+{
+ $result = array();
+
+ if ($root->hasAttributes())
+ {
+ $attrs = $root->attributes;
+
+ foreach ($attrs as $i => $attr)
+ $result[$attr->name] = $attr->value;
+ }
+
+ $children = $root->childNodes;
+if ($root->childNodes) {
+ if ($children->length == 1)
+ {
+ $child = $children->item(0);
+
+ if ($child->nodeType == XML_TEXT_NODE)
+ {
+ $result['_value'] = $child->nodeValue;
+
+ if (count($result) == 1)
+ return $result['_value'];
+ else
+ return $result;
+ }
+ }
+
+ $group = array();
+
+ for($i = 0; $i < $children->length; $i++)
+ {
+ $child = $children->item($i);
+
+ if (!isset($result[$child->nodeName]))
+ $result[$child->nodeName] = dom_to_array($child);
+ else
+ {
+ if (!isset($group[$child->nodeName]))
+ {
+ $tmp = $result[$child->nodeName];
+ $result[$child->nodeName] = array($tmp);
+ $group[$child->nodeName] = 1;
+ }
+
+ $result[$child->nodeName][] = dom_to_array($child);
+ }
+ }
+}
+
+ return $result;
+}
+
+function importCN($cnid) {
+ global $conn;
+ $CN = str_replace("-A", "00",$cnid);
+ // check if already complete
+ $query = 'Select "parentCN" from contractnotice
+ where "CNID" = :CNID';
+ $query = $conn->prepare($query);
+ $query->bindParam(":CNID", $CN);
+ $query->execute();
+ $r = $query->fetch(PDO::FETCH_ASSOC);
+ if ($r['parentCN'] == NULL) {
+$site = "https://www.tenders.gov.au/";
+$searchResult = phpQuery::newDocument(getURL("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
+//echo "https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid;
+$url = "";
+foreach(pq('a') as $a) {
+ if (strpos($a->getAttribute("href"),"public.cn.view") >0 ) {
+ //echo $a->getAttribute("href");
+ $url = $a->getAttribute("href");
+ break;
+ }
+}
+$cn = phpQuery::newDocument(getURL($site.$url));
+ $datamapping0711 = array(
+ "Agency" => "agencyName",
+ "Parent CN" => "parentCN",
+ "CN ID" => "CNID",
+ "Publish Date" => "publishDate",
+ "Amendment Date" => "amendDate",
+ "Status" => "",
+ "StartDate" => "contractStart",
+ "EndDate" => "contractEnd",
+ "Contract Value (AUD)" => "value",
+ "Description" => "description",
+ "Agency Reference ID" => "agencyID",
+ "Category" => "category",
+ "Procurement Method" => "procurementMethod",
+ "ATM ID" => "atmID",
+ "SON ID" => "SONID",
+ "Confidentiality - Contract" => "confidentialityContract",
+ "Confidentiality Reason(s) - Contract" => "confidentialityContractReason",
+ "Confidentiality - Outputs" => "confidentialityOutputs",
+ "Confidentiality Reason(s) - Outputs" => "confidentialityOutputsReason",
+ "Consultancy" => "consultancy",
+ "Consultancy Reason(s)" => "consultancyReason",
+ "Amendment Reason" => "amendmentReason",
+ "Name" => "supplierName",
+ "Postal Address" => "supplierAddress",
+ "Town/City" => "supplierCity",
+ "Postcode" => "supplierPostcode",
+ "Country" => "supplierCountry",
+ "ABN Exempt" => "supplierABNExempt",
+ "ABN" => "supplierABN",
+ "Branch" => "contactBranch",
+ "Division" => "contactDivision",
+ "Office Postcode" => "contactPostcode"
+ );
+$cnFields = Array();
+foreach(pq('tr') as $tr) {
+ $tra = dom_to_array($tr);
+ if (is_array($tra['th'])) {
+ $fieldName = trim(getTextFromTHNode($tr));
+ } else {
+ $fieldName = trim(str_replace("/th>","",$tra['th']));
+ }
+ $fieldValue = trim(print_r($tra['td'],true));
+ if ($fieldName == "State/Territory" || $fieldName == "Contact Name"
+ || $fieldName == "Contact Phone" || $fieldName == "Contact Email"
+ ||$fieldName == "Amendments") {
+ // do nothing
+ } else if ($fieldName == "Contract Period") {
+ $contractPeriod = explode("to",$fieldValue);
+ $cnFields["contractStart"] = trim($contractPeriod[0]);
+ $cnFields["contractEnd"] = trim($contractPeriod[1]);
+ } else {
+ $fieldName = $datamapping0711[$fieldName];
+
+ if ($fieldName == "parentCN" || $fieldName == "CNID") {
+ if (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+ $fieldValue = substr($fieldValue, 2); // take off the "CN" prefix
+ $fieldValue = str_replace("-A", "00", $fieldValue); // make amendments really big numbers
+ } elseif ($fieldName == "description") {
+
+ if (is_array($tra['td'])) $fieldValue = print_r($tra['td']['p'],true);
+
+ } elseif ($fieldName == "value" || $fieldName == "supplierABN") {
+ if (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+ $fieldValue = str_replace(Array("$",","," "), "", $fieldValue);
+ //if (!is_numeric($fieldValue)) $fieldValue = 0;
+ if ($fieldValue == "Exempt") $fieldValue = NULL;
+ } elseif ($fieldName == "amendDate" || $fieldName == "publishDate" || $fieldName == "contractStart" || $fieldName == "contractEnd") {
+ $fieldValue = date('Y-m-d H:i:s', strtotime($fieldValue));
+ } elseif (is_array($tra['td'])) {
+ $fieldValue = trim(getTextFromNode($tr));
+ }
+ echo $fieldName. " = " .$fieldValue."<br>\n";
+ $cnFields[$fieldName] = $fieldValue;
+ }
+}
+if (isset($cnFields[""])) {
+ $cnFields["description"] .= $cnFields[""];
+ unset($cnFields[""]);
+}
+$cnFields["importFile"] = $url;
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', array_keys($cnFields)) . '") VALUES ( ';
+ for($key = 0; $key < sizeof($cnFields); $key++) {
+ $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+ }
+ $contractNoticeInsertQ.= ");";
+ //echo $contractNoticeInsertQ;
+ $contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+ $contractNoticeInsertQ->execute(array_values($cnFields));
+ $errors = $conn->errorInfo();
+ if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+ echo "Dupe {$cnFields['CNID']}<br>";
+ }
+ elseif ($errors[1] == 0) {
+ echo "Success insert {$cnFields['CNID']} <br>";
+ }
+ else {
+ foreach ($cnFields as $key => $cnf) {
+ echo var_dump($key) . $cnf . "<br>";
+ }
+ echo $cnFields['CNID'] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+ }
+$contractNoticeUpdateQ = 'UPDATE contractnotice SET ';
+$count = 0;
+ foreach ($cnFields as $key => $f) {
+
+ $count++;
+ $contractNoticeUpdateQ.= '"'.$key.'"=? '.($count >= sizeof($cnFields) ? "" : ", ");
+ }
+ $contractNoticeUpdateQ.= ' WHERE "CNID"=?;';
+ $cnFields[] = $cnFields["CNID"];
+ //echo $contractNoticeUpdateQ;
+ $contractNoticeUpdateQ = $conn->prepare($contractNoticeUpdateQ);
+ $contractNoticeUpdateQ->execute(array_values($cnFields));
+ $errors = $conn->errorInfo();
+ if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+ print_r($errors);
+ echo "Dupe update {$cnFields['CNID']}<br>";
+ }
+ elseif ($errors[1] == 0) {
+ echo "Success update {$cnFields['CNID']} <br>";
+ }
+ else {
+ foreach ($cnFields as $key => $cnf) {
+ echo var_dump($key) . $cnf . "<br>";
+ }
+ echo $cnFields['CNID'] . " failed CN update.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+ }
+ }
+}
+function processFile($fpath, $tablename)
+{
+ global $conn;
+ echo " ============== $fpath ============== <br>";
+
+ $handle = fopen($fpath, "r");
+ flush();
+ $row = 1;
+
+ while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+ if ($row > 3) {
+ $data[0] = trim($data[0], "=");
+ $data[0] = trim($data[0], "\"");
+ if (strpos($data[0], "-A") > 0) {
+ echo "Loading {$data[0]} ... <br>\n";
+ importCN(str_replace("CN","",$data[0]));
+ }
+ }
+ flush();
+ //echo "<hr>\n";
+
+
+ $row++;
+ }
+ fclose($handle);
+}
+$path = './';
+if ($_REQUEST["fname"] == "") {
+ echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+ $dhandle = opendir($path);
+ // define an array to hold the files
+ $files = array();
+ if ($dhandle) {
+ // loop through all of the files
+ while (false !== ($fname = readdir($dhandle))) {
+ if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) {
+ echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
+ processFile($path . $fname, "contractnotice");
+ }
+ }
+ }
+}
+else {
+ $success = 0;
+ $fname = $_REQUEST["fname"];
+
+ $success+= processFile($path . $fname, "contractnotice");
+
+}
+
+?>
+
--- a/admin/partialdata/scrapesingle.php
+++ /dev/null
@@ -1,881 +1,1 @@
-<?php
-include_once ("../../lib/common.inc.php");
-$cnid = 1234;
-// http://www.lastcraft.com/browser_documentation.php
-// http://code.google.com/p/phpquery/
-require('phpQuery-onefile.php');
-function dom_to_array($root)
-{
- $result = array();
-
- if ($root->hasAttributes())
- {
- $attrs = $root->attributes;
-
- foreach ($attrs as $i => $attr)
- $result[$attr->name] = $attr->value;
- }
-
- $children = $root->childNodes;
-if ($root->childNodes) {
- if ($children->length == 1)
- {
- $child = $children->item(0);
-
- if ($child->nodeType == XML_TEXT_NODE)
- {
- $result['_value'] = $child->nodeValue;
-
- if (count($result) == 1)
- return $result['_value'];
- else
- return $result;
- }
- }
-
- $group = array();
-
- for($i = 0; $i < $children->length; $i++)
- {
- $child = $children->item($i);
-
- if (!isset($result[$child->nodeName]))
- $result[$child->nodeName] = dom_to_array($child);
- else
- {
- if (!isset($group[$child->nodeName]))
- {
- $tmp = $result[$child->nodeName];
- $result[$child->nodeName] = array($tmp);
- $group[$child->nodeName] = 1;
- }
-
- $result[$child->nodeName][] = dom_to_array($child);
- }
- }
-}
-
- return $result;
-}
-
-$site = "https://www.tenders.gov.au/";
-//$cn = phpQuery::newDocument(file_get_contents("https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN".$cnid));
-$searchResult = phpQuery::newDocument('<table class="four-col">
- <tbody><tr>
- <th>CN ID</th>
-
- <td><a href="/?event=public.cn.view&CNUUID=BB3B13EA-A700-39A5-F003088ACF798AF9">CN1234</a></td>
-
- <th>
- </th><td>
-
- </td></tr>
- <tr>
- <th>Agency</th>
- <td colspan="3">Department of Veterans Affairs</td>
- </tr>
- <tr>
- <th>Publish Date</th>
- <td colspan="3"> 18-Aug-2006 </td>
- </tr>
-
- <tr>
- <th>Category</th>
- <td colspan="3">Computer services</td>
- </tr>
-
- <tr>
- <th>Contract Period</th>
- <td colspan="3"> 1-Aug-2006 to 31-Jul-2007 </td>
- </tr>
- <tr>
- <th>Contract Value (AUD)</th>
- <td colspan="3"> $156,200.00 </td>
- </tr>
-
- <tr>
- <th>Supplier Name</th>
- <td colspan="3">WIZARD INFORMATION SERVICES PTY LTD</td>
- </tr>
- <tr>
- <th></th>
- <td colspan="3">
- <p>
- <span class="last-updated"><strong>Last Updated:</strong> 18