remove amon, fix openid
--- a/admin/data/CN806771.2cb256c016c609172a97a04741a278f8.xls
+++ b/admin/data/CN806771.2cb256c016c609172a97a04741a278f8.xls
@@ -1010,7 +1010,7 @@
="Future Fund Management Agency" ="CN987101" ="CN987101-A2" 25-Oct-12 09:58 AM 19-Dec-12 10:44 AM ="published" 10-Oct-12 31-Mar-13 67000.00 " Contractor fee " ="FFMA0513" ="Temporary personnel services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "Contract extension" ="Ranstad Pty Ltd" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="28080275378" ="" ="" ="3000"
="Future Fund Management Agency" ="" ="CN1104112" 19-Dec-12 10:47 AM ="published" 26-Nov-12 01-Mar-13 15000.00 " Contractor fees " ="FFMA0527" ="Temporary personnel services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Randstad" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="28080275378" ="" ="" ="3000"
="Australian Securities and Investments Commission" ="" ="CN1104152" 19-Dec-12 09:50 AM ="published" 12-Apr-12 30-Jun-13 178970.00 " Vetting " ="2854" ="Security surveillance and detection" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Premier Protection Agency" ="PO Box 3401" ="Liverpool" ="2170" ="AUSTRALIA" ="No" ="37078491826" ="" ="" ="2000"
-="Department of Defence" ="" ="CN1104192" 19-Dec-12 10:01 AM ="published" 22-Jul-09 30-Sep-09 26180.00 " MRH 90 PROJECT NET ACTIONS SCRIPT PROGRAMMER " ="4500714409" ="Computer services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="DESIGN AND BUILD THIS" ="14 LARKEN AVE" ="BAULKHAM HILLS" ="2153" ="AUSTRALIA" ="No" ="93223025707" ="ARMY Headquarters" ="Army Group" ="2000"
+="Department of Defence" ="" ="CN1104192" 19-Dec-12 10:01 AM ="published" 22-Jul-09 30-Sep-09 26180.00 " MRH 90 PROJECT NET ACTIONS SCRIPT PROGRAMMER " ="4500714409" ="Computer services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="DESIGN AND BUILD THIS" ="" ="BAULKHAM HILLS" ="2153" ="AUSTRALIA" ="No" ="93223025707" ="ARMY Headquarters" ="Army Group" ="2000"
="Department of Industry, Innovation, Science, Research and Tertiary Education" ="CN1065821" ="CN1065821-A1" 03-Dec-12 02:22 PM 19-Dec-12 10:06 AM ="published" 07-Aug-12 30-Jun-13 13491.50 " HTML Conversion of 2011-12 Annual Report " ="CONTRACTS-004828-0" ="Internet services" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "Increase contract value" ="SWELL DESIGN GROUP" ="" ="MANUKA" ="2603" ="AUSTRALIA" ="No" ="95060811725" ="" ="" ="2601"
="Defence Materiel Organisation" ="" ="CN1104232" 19-Dec-12 10:06 AM ="published" 19-Dec-12 21-May-13 87747.00 " purchase of test sets and shipping crates " ="NC1YY9" ="Aerospace systems and components and equipment" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="astrite pty ltd" ="5 jarrah court" ="hoppers crossing" ="3029" ="AUSTRALIA" ="No" ="18756753773" ="" ="" ="3027"
="Future Fund Management Agency" ="" ="CN1104262" 19-Dec-12 10:46 AM ="published" 30-Nov-12 30-Nov-12 31680.00 " Personnel Recruitment " ="FFMA0526" ="Personnel recruitment" ="Limited tender" ="" ="No" ="" ="No" ="" ="No" ="" "" ="Mahlad Recruitment Pty Lts" ="" ="Melbourne" ="3000" ="AUSTRALIA" ="No" ="46633561134" ="" ="" ="3000"
@@ -1847,4 +1847,5 @@
="Office of the Director of Public Prosecutions" ="" ="CN1112292" 21-Dec-12 06:59 PM ="published" 01-Jan-13 31-Oct-13 120000.00 " Estimated value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Probe Group Pty Ltd" ="" ="Melbourne" ="3000" ="AUSTRALIA" ="No" ="90770747494" ="" ="" ="2601"
="Office of the Director of Public Prosecutions" ="" ="CN1112302" 21-Dec-12 07:02 PM ="published" 01-Jan-13 31-Oct-13 210000.00 " Estimated Value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Statewide Mercantile Services Pty Ltd" ="" ="West Melbourne" ="3000" ="AUSTRALIA" ="No" ="77075475544" ="" ="" ="2601"
="Office of the Director of Public Prosecutions" ="" ="CN1112312" 21-Dec-12 07:04 PM ="published" 01-Jan-13 31-Oct-13 205000.00 " Estimated value for process servers " ="" ="Legal services" ="Open tender" ="" ="SON1112262" ="No" ="" ="No" ="" ="No" ="" "" ="Wise McGrath" ="" ="Sydney" ="2000" ="AUSTRALIA" ="No" ="14096930568" ="" ="" ="2601"
-="Australian Law Reform Commission" ="" ="CN1014921" 17-Dec-12 02:20 PM ="published" 01-Nov-12 02-Nov-15 12000.00 " Office supplies and stationary " ="ALRC 02" ="Stationery" ="Open tender" ="FIN11/FMG010" ="SON473022" ="No" ="" ="No" ="" ="No" ="" "" ="OfficeMax Australia Limited" ="20 Worth St" ="Chullora" ="2190" ="AUSTRALIA" ="No" ="26064777224" ="Australian Law Reform Commission" ="Attorney-General's Department" ="2001"
+="Australian Law Reform Commission" ="" ="CN1014921" 17-Dec-12 02:20 PM ="published" 01-Nov-12 02-Nov-15 12000.00 " Office supplies and stationary " ="ALRC 02" ="Stationery" ="Open tender" ="FIN11/FMG010" ="SON473022" ="No" ="" ="No" ="" ="No" ="" "" ="OfficeMax Australia Limited" ="20 Worth St" ="Chullora" ="2190" ="AUSTRALIA" ="No" ="26064777224" ="Australian Law Reform Commission" ="Attorney-General's Department" ="2001"
+
--- a/admin/import.php
+++ b/admin/import.php
@@ -86,7 +86,8 @@
"Agency Postcode" => "contactPostcode",
"" => ""
);
-
+
+ $headers;
while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
$num = count($data);
@@ -171,7 +172,7 @@
}
ksort($files);
foreach ($files as $date => $fname) {
- echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
+ echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
}
} else {
$success = 0;
--- a/admin/importdatagov.php
+++ /dev/null
@@ -1,216 +1,1 @@
-<?php
-include_once ("../lib/common.inc.php");
-auth();
-/* todo gaps
-Contract ID,Department,Portfolio,Division,Branch,Agency Ref,Contract Date,End Date,Value,Office PCode,Description,Procurement Desc,Confidentiality Reason,Consultancy,ANZSCC Code,ANSCC Desc,Supplier,Supplier Postal Addr,Supplier Suburb,Supplier State,Supplier Country,Supplier PCode,Supplier ABN,Supplier DUNS,Supplier ACN,S/O Reference Num
-1694570,Department of Defence,Defence,NEW SOUTH WALES,RAN HMAS WATSON,1906458734 ,30-Jun-2007,30-Jun-2007,16083.96,2612,AIRLINE TICKETS,direct source,,No,731,Passenger transportation by air,QANTAS AIRWAYS LTD,PO Box PB 747,MASCOT,NSW,Australia,2020,16009661901,750512642,009661901 ,
-
-*/
-$contractNoticeFields = array(
- "importFile",
- "agencyName",
- "parentCN",
- "CNID",
- "publishDate",
- "amendDate",
- "contractStart",
- "contractEnd",
- "value",
- "description",
- "agencyID",
- "categoryUNSPSC",
- "category",
- "procurementMethod",
- "atmID",
- "SONID",
- "confidentialityContract",
- "confidentialityContractReason",
- "confidentialityOutputs",
- "confidentialityOutputsReason",
- "consultancy",
- "consultancyReason",
- "amendmentReason",
- "supplierName",
- "supplierAddress",
- "supplierCity",
- "supplierPostcode",
- "supplierCountry",
- "supplierABNExempt",
- "supplierABN",
- "contactBranch",
- "contactDivision",
- "contactPostcode"
-);
-
-$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
-foreach ($contractNoticeFields as $key => $f) {
- $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
-}
-$contractNoticeInsertQ.= ");";
-$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
-
-function processFile($fpath) {
- global $conn, $contractNoticeFields, $contractNoticeInsertQ;
- $row = 1;
- $success = 0;
- ini_set('auto_detect_line_endings',TRUE);
- $handle = fopen($fpath, "r");
- //"t" mode string translates windows line breaks to unix
- $datamapping0712 = array(
-
- "Agency Name" => "agencyName",
- "Parent Contract ID" => "parentCN",
- "Contract ID" => "CNID",
- "Publish Date" => "publishDate",
- "Amendment Date" => "amendDate",
- "Start Date" => "contractStart",
- "End Date" => "contractEnd",
- "Value" => "value",
- "Description" => "description",
- "Agency Ref ID" => "agencyID",
- "UNSPSC Code" => "categoryUNSPSC",
- "Title" => "category",
- "Procurement Method" => "procurementMethod",
- "ATM ID" => "atmID",
- "SON ID" => "SONID",
- "Confidentiality Contract Flag" => "confidentialityContract",
- "Confidentiality Contract Reason" => "confidentialityContractReason",
- "Confidentiality Outputs Flag" => "confidentialityOutputs",
- "Confidentiality Outputs Reason" => "confidentialityOutputsReason",
- "Consultancy Flag" => "consultancy",
- "Consultancy Reason" => "consultancyReason",
- "Amendment Reason" => "amendmentReason",
- "Supplier Name" => "supplierName",
- "Supplier Address" => "supplierAddress",
- "Supplier Suburb" => "supplierCity",
- "Supplier Postcode" => "supplierPostcode",
- "Supplier Country" => "supplierCountry",
- "Supplier ABN Exempt" => "supplierABNExempt",
- "ABN" => "supplierABN",
- "Contact Name" => "",
- "Contact Phone" => "",
- "Branch" => "contactBranch",
- "Division" => "contactDivision",
- "Office Postcode" => "contactPostcode",
-
- );
-
-
- while (($data = fgetcsv($handle, 10000)) !== false) {
- //print_r($data);
- $num = count($data);
- if ($row == 1) {
- $headers = $data;
- } elseif ($row > 1) {
- if ($num > count($datamapping0712)) {
- die("<font color=red>Error in data import; data mapping fields out of bounds or changed $num > ".count($datamapping0712)."</font><br>" . $fpath . print_r($data));
- }
- $contractNoticeInsert = Array();
- $supplierInsert = Array();
- $agencyInsert = Array();
- $contractNoticeInsert[] = $fpath;
- $keys = array_keys($datamapping0712);
- for ($c = 0; $c < $num; $c++) {
- $data[$c] = trim($data[$c], "=");
- $data[$c] = trim($data[$c], "\"");
- if (in_array(($datamapping0712[$headers[$c]]), $contractNoticeFields)) {
- if (($datamapping0712[$headers[$c]]) == "parentCN" || ($datamapping0712[$headers[$c]]) == "CNID") {
- $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
- if ($data[$c] > 0 && $data[$c] != '0') {
- $contractNoticeInsert[] = $data[$c];
- } else {
- $contractNoticeInsert[] = null;
- }
- } elseif (($datamapping0712[$headers[$c]]) == "supplierABN") {
- if ($data[$c] > 0 && $data[$c] != '0') {
- $contractNoticeInsert[] = $data[$c];
- } else {
- $contractNoticeInsert[] = null;
- }
- } elseif (($datamapping0712[$headers[$c]]) == "amendDate" || ($datamapping0712[$headers[$c]]) == "publishDate" || ($datamapping0712[$headers[$c]]) == "contractStart" || ($datamapping0712[$headers[$c]]) == "contractEnd") {
- $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
- } else {
- if (strstr("\" =", $data[$c] > 0)) {
- die("Invalid Description field" . $contractNoticeInsert);
- }
- $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));
-
- $contractNoticeInsert[] = $colvalue;
- }
- }
- }
- flush();
- //print_r($contractNoticeInsert);
- $contractNoticeInsertQ->execute($contractNoticeInsert);
- $errors = $conn->errorInfo();
- if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
- // echo "dupe <br>";
- } elseif ($errors[1] == 0) {
- $success++;
- } else {
- foreach ($contractNoticeFields as $key => $cnf) {
- echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
- }
- echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
- }
-
- flush();
- //echo "<hr>\n";
- }
- $row++;
- }
- fclose($handle);
- $contractNoticeInsertQ->closeCursor();
-
- return $success;
-}
-
-$path = 'datagovdata/';
-if ($_REQUEST["fname"] == "" && $argv[1] == "") {
- echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
- $dhandle = opendir($path);
- // define an array to hold the files
- $files = array();
- if ($dhandle) {
- // loop through all of the files
- while (false !== ($fname = readdir($dhandle))) {
- if (($fname != '.') && ($fname != '..')) {
- $files[date("c", filemtime($path . $fname)) . md5($fname)] = $fname;
- }
- }
- }
- ksort($files);
- foreach ($files as $date => $fname) {
- echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . $date . "<br/>";
- }
-} else {
- $success = 0;
- $fname = $_REQUEST["fname"];
- if ($argv[1] != "") $fname = $argv[1];
- echo " ============== $fname ============== <br>";
- flush();
- $success+= processFile($path . $fname, "contractnotice");
- $success+= processFile($path . $fname, "agency");
- $success+= processFile($path . $fname, "supplier");
- echo "<br> $success records successfully created";
-
- flush();
- // run post import data processing
- //
- if ($success > 0) {
-$conn->exec("update datasets set \"lastUpdated\" = NOW() where title = 'Contract Notices'");
- echo "link amend<br>";
- include ("linkAmendments.php");
- echo "update UNSPSC<br>";
- include ("updateUNSPSC.php");
- }
-// cn
-
-// agency
-//include ("setAgencyStatus.php");
-//include ("setAgencyURLABN.php");
-
-}
-?>
-
--- a/admin/neo4jimporter/pom.xml
+++ b/admin/neo4jimporter/pom.xml
@@ -7,13 +7,12 @@
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId>
- <version>2.0.0-M03</version>
+ <version>1.8.RC1</version>
</dependency>
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>9.1-901.jdbc4</version>
+ <version>9.0-801.jdbc4</version>
</dependency>
</dependencies>
</project>
-
--- a/admin/neo4jimporter/src/main/java/Importer.java
+++ b/admin/neo4jimporter/src/main/java/Importer.java
@@ -5,34 +5,25 @@
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
+import java.util.HashMap;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Writer;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.neo4j.graphdb.Direction;
-import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService;
-import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
-import org.neo4j.graphdb.RelationshipType;
+import org.neo4j.graphdb.index.BatchInserterIndex;
+import org.neo4j.graphdb.index.BatchInserterIndexProvider;
import org.neo4j.helpers.collection.MapUtil;
-import org.neo4j.unsafe.batchinsert.BatchInserter;
-import org.neo4j.unsafe.batchinsert.BatchInserters;
+import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
+import org.neo4j.kernel.impl.batchinsert.BatchInserter;
+import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
public class Importer {
public static void main(String[] argv) {
-Map<String, String> config = new HashMap<String, String>();
-config.put( "neostore.nodestore.db.mapped_memory", "90M" );
-BatchInserter inserter = BatchInserters.inserter("target/batchinserter-example-config", config );
- //BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
- //BatchInserterIndex names = indexProvider.nodeIndex("names", MapUtil.stringMap("type", "exact"));
- //names.setCacheCapacity("name", 100000);
+ BatchInserter inserter = new BatchInserterImpl("target/neo4jdb-batchinsert");
+ BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);
+ BatchInserterIndex labels = indexProvider.nodeIndex("labels", MapUtil.stringMap("type", "exact"));
+ labels.setCacheCapacity("Label", 100000);
@@ -87,56 +78,52 @@
// Execute the query
ResultSet rs = stmt.executeQuery("SELECT contractnotice.\"agencyName\", "
- + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END) as supplierID , max(contractnotice.\"supplierName\") as \"supplierName\",sum(value) as sum "
+ + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\",sum(value) as sum "
+ "FROM public.contractnotice GROUP BY contractnotice.\"agencyName\", "
- + " (case when \"supplierABN\" != 0 THEN \"supplierABN\"::text ELSE \"supplierName\" END)");
+ + " contractnotice.\"supplierABN\",contractnotice.\"supplierName\"");
+ String previousAgency = "";
+ GraphDatabaseService gds = inserter.getGraphDbService();
HashMap<String, Long> supplierIDs = new HashMap<String, Long>();
HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
-
-Label agencyLabel = DynamicLabel.label( "Agency" );
-inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" );
-Label supplierLabel = DynamicLabel.label( "Supplier" );
-inserter.createDeferredSchemaIndex( agencyLabel ).on( "name" );
// Loop through the result set
while (rs.next()) {
long supplierID, agencyID;
String supplierKey;
if (agencyIDs.get(rs.getString("agencyName")) == null) {
- Map<String, Object> properties = new HashMap<String, Object>();
- properties.put("name", rs.getString("agencyName"));
- properties.put("type", rs.getString("agency"));
- agencyID = inserter.createNode(properties, agencyLabel);
- agencyIDs.put(rs.getString("agencyName"), agencyID);
- if (agencyID % 10 == 0) {
- System.out.println("Agency " + agencyID);
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("agencyName"));
+ myNode.setProperty("type", "agency");
+ agencyIDs.put(rs.getString("agencyName"), myNode.getId());
+ if (myNode.getId() % 100 == 0) {
+ System.out.println("Agency " + myNode.getId());
}
}
agencyID = agencyIDs.get(rs.getString("agencyName"));
+ if (rs.getString("supplierABN") != "0" && rs.getString("supplierABN") != "") {
+ supplierKey = rs.getString("supplierABN");
+ } else {
+ supplierKey = rs.getString("supplierName");
+ }
// inject some data
- if (supplierIDs.get(rs.getString("supplierID")) == null) {
- Map<String, Object> properties = new HashMap<String, Object>();
- properties.put("name", rs.getString("supplierName"));
- properties.put("type", rs.getString("supplier"));
- supplierID = inserter.createNode(properties, supplierLabel);
- supplierIDs.put(rs.getString("supplierID"), supplierID);
- if (supplierID % 1000 == 0) {
- System.out.println("Supplier " + supplierID);
+ if (supplierIDs.get(supplierKey) == null) {
+ Node myNode = gds.createNode();
+ myNode.setProperty("Label", rs.getString("supplierName"));
+ myNode.setProperty("type", "supplier");
+ supplierIDs.put(supplierKey, myNode.getId());
+ if (myNode.getId() % 1000 == 0) {
+ System.out.println("Supplier " + myNode.getId());
}
}
- supplierID = supplierIDs.get(rs.getString("supplierID"));
+ supplierID = supplierIDs.get(supplierKey);
-// To set properties on the relationship, use a properties map
-// instead of null as the last parameter.
-Map<String, Object> properties = new HashMap<String, Object>();
-properties.put( "value", rs.getDouble("sum"));
- inserter.createRelationship(agencyID, supplierID,
- DynamicRelationshipType.withName("PAYS"), properties);
- inserter.createRelationship(supplierID, agencyID,
- DynamicRelationshipType.withName("PAID_BY"), properties);
+ long rel = inserter.createRelationship(agencyID, supplierID,
+ DynamicRelationshipType.withName("KNOWS"), null);
+ inserter.setRelationshipProperty(rel, "Weight", rs.getDouble("sum"));
+
}
// Close the result set, statement and the connection
rs.close();
@@ -155,11 +142,10 @@
}
}
//make the changes visible for reading, use this sparsely, requires IO!
-// names.flush();
+ labels.flush();
// Make sure to shut down the index provider
-// indexProvider.shutdown();
+ indexProvider.shutdown();
inserter.shutdown();
}
-}
-
+}
Binary files a/admin/neo4jimporter/target/classes/Importer.class and b/admin/neo4jimporter/target/classes/Importer.class differ
--- a/admin/partialdata/import.php
+++ b/admin/partialdata/import.php
@@ -12,7 +12,6 @@
$handle = fopen($fpath, "r");
//"t" mode string translates windows line breaks to unix
$datamapping0507 = array(
-
"Agency" => "agencyName",
"CN ID" => "CNID",
"Publish Date" => "publishDate",
@@ -138,7 +137,7 @@
// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..') && (!isset($_REQUEST["filter"]) || strpos($fname,$_REQUEST["filter"]) != false)) {
- echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
+ echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice");
}
}
--- a/admin/partialdata/importamendments.php
+++ b/admin/partialdata/importamendments.php
@@ -310,7 +310,7 @@
// loop through all of the files
while (false !== ($fname = readdir($dhandle))) {
if (($fname != '.') && ($fname != '..') && (strpos($fname,".xls")>0)) {
- echo "<a href=\"importdatagov.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
+ echo "<a href=\"import.php?fname=$fname\">$fname</a> " . filesize($path . $fname) . " " . date("c", filemtime($path . $fname)) . "<br/>";
processFile($path . $fname, "contractnotice");
}
}
--- a/exportOverview.csv.php
+++ b/exportOverview.csv.php
@@ -11,11 +11,10 @@
$unspsc[$row['UNSPSC']] = $row['Title'];
}
-//(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url
$query = $conn->prepare('
-SELECT "CNID" as id, description as text
-from "contractnotice"
-where "childCN" is null limit 10000'
+SELECT "CNID" as uid, description as text,
+(\'https://www.tenders.gov.au/?event=public.advancedsearch.keyword&keyword=CN\'::text || "CNID"::text) as url from "contractnotice"
+where "childCN" is null'
, array(PDO::ATTR_CURSOR => PDO::FETCH_ORI_NEXT));
$query->execute();
$errors = $conn->errorInfo();
--- a/lib/common.inc.php
+++ b/lib/common.inc.php
@@ -14,14 +14,6 @@
if (strstr($_SERVER['PHP_SELF'], "labs/") || strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "heuristics/")) {
$basePath = "../";
}
-if ($_SERVER['HTTP_HOST'] != "localhost") {
-require $basePath."lib/amon-php/amon.php";
-Amon::config(array('address'=> 'http://127.0.0.1:2464',
- 'protocol' => 'http',
- 'secret_key' => "JBcSUdFOi5lK0vCjLjbHDpQamcBnRA4iV7QLaTADeDQ"));
-Amon::setup_exception_handler();
-}
-
require ROOT . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'lib' . DIRECTORY_SEPARATOR . 'openid.php';
$openid = new LightOpenID($_SERVER['HTTP_HOST']);
// you have to open the session to be able to modify or remove it
@@ -314,3 +306,6 @@
include ("graphs.inc.php");
+
+
+
--- a/lib/openid.php
+++ b/lib/openid.php
@@ -81,7 +81,7 @@
$this->trustRoot = (strpos($host, '://') ? $host : 'https://' . $host);
}
- if(($host_end = strpos($this->trustRoot, '/', 8)) !== false) {
+ if(($host_end = @strpos($this->trustRoot, '/', 8)) !== false) {
$this->trustRoot = substr($this->trustRoot, 0, $host_end);
}