projects / contractdashboard.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
side by side | plain
Update import for better PDO statement reuse and only store name/abn/status for agencies
Update import for better PDO statement reuse and only store name/abn/status for agencies

file:a/admin/fixoldamend.php -> file:b/admin/fixoldamend.php 
--- a/admin/fixoldamend.php
+++ b/admin/fixoldamend.php
@@ -20,7 +20,7 @@
     echo "$oldCN => $newCN (from parent CN $parentCN) <br>\n";
     }
     else {
-        echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n";
+//        echo "parent CN unexpected - $oldCN doesn't look like child of {$record['parentCN']} <br>\n";
     }
 }
file:a/admin/import.php -> file:b/admin/import.php 
--- a/admin/import.php
+++ b/admin/import.php
@@ -1,271 +1,258 @@
 <?php
+
 include_once ("../lib/common.inc.php");
-function processFile($fpath, $tablename)
-{
-	global $conn;
-	$row = 1;
-	$handle = fopen($fpath, "r");
-	//"t" mode string translates windows line breaks to unix
-	$datamapping0711 = array(
-		"Agency" => "agencyName",
-		"Parent CN ID" => "parentCN",
-		"CN ID" => "CNID",
-		"Publish Date" => "publishDate",
-		"Amendment Date" => "amendDate",
-		"Status" => "",
-		"StartDate" => "contractStart",
-		"EndDate" => "contractEnd",
-		"Value" => "value",
-		"Description" => "description",
-		"Agency Ref Id" => "agencyID",
-		"Category" => "category",
-		"Procurement Method" => "procurementMethod",
-		"ATM ID" => "atmID",
-		"SON ID" => "SONID",
-		"Confidentiality - Contract" => "confidentialityContract",
-		"Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
-		"Confidentiality - Outputs" => "confidentialityOutputs",
-		"Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
-		"Consultancy" => "consultancy",
-		"Consultancy Reason(s)" => "consultancyReason",
-		"Amendment Reason" => "amendmentReason",
-		"Supplier Name" => "supplierName",
-		"Supplier Address" => "supplierAddress",
-		"Supplier City" => "supplierCity",
-		"Supplier Postcode" => "supplierPostcode",
-		"Supplier Country" => "supplierCountry",
-		"Supplier ABNExempt" => "supplierABNExempt",
-		"Supplier ABN" => "supplierABN",
-		"Agency Branch" => "contactBranch",
-		"Agency Divison" => "contactDivision",
-		"Agency Postcode" => "contactPostcode",
-		"" => ""
-	);
-	$headers;
-	$contractNoticeFields = array(
-		"importFile",
-		"agencyName",
-		"parentCN",
-		"CNID",
-		"publishDate",
-		"amendDate",
-		"contractStart",
-		"contractEnd",
-		"value",
-		"description",
-		"agencyID",
-		"category",
-		"procurementMethod",
-		"atmID",
-		"SONID",
-		"confidentialityContract",
-		"confidentialityContractReason",
-		"confidentialityOutputs",
-		"confidentialityOutputsReason",
-		"consultancy",
-		"consultancyReason",
-		"amendmentReason",
-		"supplierName",
-		"supplierAddress",
-		"supplierCity",
-		"supplierPostcode",
-		"supplierCountry",
-		"supplierABNExempt",
-		"supplierABN",
-		"contactBranch",
-		"contactDivision",
-		"contactPostcode"
-	);
-	$agencyFields = array(
-		"agencyName"
-	);
-	$supplierFields = array(
-		"supplierName",
-		"supplierAddress",
-		"supplierCity",
-		"supplierPostcode",
-		"supplierCountry",
-		"supplierABNExempt",
-		"supplierABN"
-	);
-	if ($tablename == "contractnotice") {
-		$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
-		foreach ($contractNoticeFields as $key => $f) {
-			$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
-		}
-		$contractNoticeInsertQ.= ");";
-		$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
-	}
-	else if ($tablename == "supplierdetails") {
-		$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
-		foreach ($supplierFields as $key => $f) {
-			$supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
-		}
-		$supplierInsertQ.= ");";
-		$supplierInsertQ = $conn->prepare($supplierInsertQ);
-	}
-	else if ($tablename == "agency") {
-		$agencyInsertQ = 'INSERT INTO agency ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
-		foreach ($agencyFields as $key => $f) {
-			$agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
-		}
-		$agencyInsertQ.= ");";
-		$agencyInsertQ = $conn->prepare($agencyInsertQ);
-	}
-	while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
-		$num = count($data);
-		if ($row == 3) {
-			$headers = $data;
-		}
-		elseif ($row > 3) {
-			if ($num > count($datamapping0711)) {
-				die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
-			}
-			$contractNoticeInsert = Array();
-			$supplierInsert = Array();
-			$agencyInsert = Array();
-			$contractNoticeInsert[] = $fpath;
-			$keys = array_keys($datamapping0711);
-			for ($c = 0; $c < $num; $c++) {
-				$data[$c] = trim($data[$c], "=");
-				$data[$c] = trim($data[$c], "\"");
-				if ($tablename == "contractnotice") {
-					if (in_array(($datamapping0711[$headers[$c]]) , $contractNoticeFields)) {
-						if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
-							$data[$c] = substr($data[$c], 2); // take off the "CN" prefix
-							if ($data[$c] > 0) {
-								$contractNoticeInsert[] = $data[$c];
-							}
-							else {
-								$contractNoticeInsert[] = 0;
-							}
-						}
-						elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
-							if ($data[$c] > 0) {
-								$contractNoticeInsert[] = $data[$c];
-							}
-							else {
-								$contractNoticeInsert[] = null;
-							}
-						}
-						elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
-							$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
-						}
-						else {
-							if (strstr("\" =", $data[$c] > 0)) {
-								die("Invalid Description field" . $contractNoticeInsert);
-							}
-$colvalue =  preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));
-
-							$contractNoticeInsert[] = $colvalue;
-						}
-					}
-				}
-				else if ($tablename == "supplierdetails") {
-					if (in_array(($datamapping0711[$headers[$c]]) , $supplierFields)) {
-						if (($datamapping0711[$headers[$c]]) == "supplierABN") {
-							if ($data[$c] > 0) {
-								$contractNoticeInsert[] = $data[$c];
-							}
-							else {
-								$contractNoticeInsert[] = 0;
-							}
-						}
-						else {
-							$supplierInsert[] = $data[$c];
-						}
-					}
-				}
-				else if ($tablename == "agency") {
-					if (in_array(($datamapping0711[$headers[$c]]) , $agencyFields)) {
-						$agencyInsert[] = $data[$c];
-					}
-				}
-			}
-			flush();
-			if ($tablename == "contractnotice") {
-				$contractNoticeInsertQ->execute($contractNoticeInsert);
-				$errors = $conn->errorInfo();
-				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
-				}
-				elseif ($errors[1] == 0) {
-					$success++;
-				}
-				else {
-					foreach ($contractNoticeFields as $key => $cnf) {
-						echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
-					}
-					echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
-				}
-			}
-			else if ($tablename == "supplierdetails") {
-				$supplierInsertQ->execute($supplierInsert);
-				$errors = $conn->errorInfo();
-				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
-				}
-				elseif ($errors[1] == 0) {
-					$success++;
-				}
-				else {
-					echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
-				}
-			}
-			else if ($tablename == "agency") {
-				$agencyInsertQ->execute($agencyInsert);
-				$errors = $conn->errorInfo();
-				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
-				}
-				elseif ($errors[1] == 0) {
-					$success++;
-				}
-				else {
-					echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
-				}
-			}
-			flush();
-			//echo "<hr>\n";
-			
-		}
-		$row++;
-	}
-	fclose($handle);
-        // run post import data processing
+
+$contractNoticeFields = array(
+    "importFile",
+    "agencyName",
+    "parentCN",
+    "CNID",
+    "publishDate",
+    "amendDate",
+    "contractStart",
+    "contractEnd",
+    "value",
+    "description",
+    "agencyID",
+    "category",
+    "procurementMethod",
+    "atmID",
+    "SONID",
+    "confidentialityContract",
+    "confidentialityContractReason",
+    "confidentialityOutputs",
+    "confidentialityOutputsReason",
+    "consultancy",
+    "consultancyReason",
+    "amendmentReason",
+    "supplierName",
+    "supplierAddress",
+    "supplierCity",
+    "supplierPostcode",
+    "supplierCountry",
+    "supplierABNExempt",
+    "supplierABN",
+    "contactBranch",
+    "contactDivision",
+    "contactPostcode"
+);
+$agencyFields = array(
+    "agencyName"
+);
+$supplierFields = array(
+    "supplierName",
+    "supplierAddress",
+    "supplierCity",
+    "supplierPostcode",
+    "supplierCountry",
+    "supplierABNExempt",
+    "supplierABN"
+);
+$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
+foreach ($contractNoticeFields as $key => $f) {
+    $contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$contractNoticeInsertQ.= ");";
+$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+
+$supplierInsertQ = 'INSERT INTO supplierdetails ("' . implode('" , "', $supplierFields) . '") VALUES ( ';
+foreach ($supplierFields as $key => $f) {
+    $supplierInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$supplierInsertQ.= ");";
+$supplierInsertQ = $conn->prepare($supplierInsertQ);
+
+$agencyInsertQ = 'INSERT INTO agency_nametoabn ("' . implode('" , "', $agencyFields) . '") VALUES ( ';
+foreach ($agencyFields as $key => $f) {
+    $agencyInsertQ.= ($key == 0 ? "" : ", ") . "?";
+}
+$agencyInsertQ.= ");";
+$agencyInsertQ = $conn->prepare($agencyInsertQ);
+
+function processFile($fpath, $tablename) {
+    global $conn, $contractNoticeQ, $supplierInsertQ, $agencyInsertQ;
+    $row = 1;
+    $handle = fopen($fpath, "r");
+    //"t" mode string translates windows line breaks to unix
+    $datamapping0711 = array(
+        "Agency" => "agencyName",
+        "Parent CN ID" => "parentCN",
+        "CN ID" => "CNID",
+        "Publish Date" => "publishDate",
+        "Amendment Date" => "amendDate",
+        "Status" => "",
+        "StartDate" => "contractStart",
+        "EndDate" => "contractEnd",
+        "Value" => "value",
+        "Description" => "description",
+        "Agency Ref Id" => "agencyID",
+        "Category" => "category",
+        "Procurement Method" => "procurementMethod",
+        "ATM ID" => "atmID",
+        "SON ID" => "SONID",
+        "Confidentiality - Contract" => "confidentialityContract",
+        "Confidentiality - Contract Reason(s)" => "confidentialityContractReason",
+        "Confidentiality - Outputs" => "confidentialityOutputs",
+        "Confidentiality - Outputs Reason(s)" => "confidentialityOutputsReason",
+        "Consultancy" => "consultancy",
+        "Consultancy Reason(s)" => "consultancyReason",
+        "Amendment Reason" => "amendmentReason",
+        "Supplier Name" => "supplierName",
+        "Supplier Address" => "supplierAddress",
+        "Supplier City" => "supplierCity",
+        "Supplier Postcode" => "supplierPostcode",
+        "Supplier Country" => "supplierCountry",
+        "Supplier ABNExempt" => "supplierABNExempt",
+        "Supplier ABN" => "supplierABN",
+        "Agency Branch" => "contactBranch",
+        "Agency Divison" => "contactDivision",
+        "Agency Postcode" => "contactPostcode",
+        "" => ""
+    );
+    $headers;
+
+    while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+        $num = count($data);
+        if ($row == 3) {
+            $headers = $data;
+        } elseif ($row > 3) {
+            if ($num > count($datamapping0711)) {
+                die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
+            }
+            $contractNoticeInsert = Array();
+            $supplierInsert = Array();
+            $agencyInsert = Array();
+            $contractNoticeInsert[] = $fpath;
+            $keys = array_keys($datamapping0711);
+            for ($c = 0; $c < $num; $c++) {
+                $data[$c] = trim($data[$c], "=");
+                $data[$c] = trim($data[$c], "\"");
+                if ($tablename == "contractnotice") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $contractNoticeFields)) {
+                        if (($datamapping0711[$headers[$c]]) == "parentCN" || ($datamapping0711[$headers[$c]]) == "CNID") {
+                            $data[$c] = substr($data[$c], 2); // take off the "CN" prefix
+                            if ($data[$c] > 0) {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = 0;
+                            }
+                        } elseif (($datamapping0711[$headers[$c]]) == "supplierABN") {
+                            if ($data[$c] > 0) {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = null;
+                            }
+                        } elseif (($datamapping0711[$headers[$c]]) == "amendDate" || ($datamapping0711[$headers[$c]]) == "publishDate" || ($datamapping0711[$headers[$c]]) == "contractStart" || ($datamapping0711[$headers[$c]]) == "contractEnd") {
+                            $contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
+                        } else {
+                            if (strstr("\" =", $data[$c] > 0)) {
+                                die("Invalid Description field" . $contractNoticeInsert);
+                            }
+                            $colvalue = preg_replace('/[^[:print:]]/', '', utf8_encode($data[$c]));
+
+                            $contractNoticeInsert[] = $colvalue;
+                        }
+                    }
+                } else if ($tablename == "supplierdetails") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $supplierFields)) {
+                        if (($datamapping0711[$headers[$c]]) == "supplierABN") {
+                            if ($data[$c] > 0) {
+                                $contractNoticeInsert[] = $data[$c];
+                            } else {
+                                $contractNoticeInsert[] = 0;
+                            }
+                        } else {
+                            $supplierInsert[] = $data[$c];
+                        }
+                    }
+                } else if ($tablename == "agency") {
+                    if (in_array(($datamapping0711[$headers[$c]]), $agencyFields)) {
+                        $agencyInsert[] = $data[$c];
+                    }
+                }
+            }
+            flush();
+            if ($tablename == "contractnotice") {
+                $contractNoticeInsertQ->execute($contractNoticeInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    foreach ($contractNoticeFields as $key => $cnf) {
+                        echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
+                    }
+                    echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+                }
+            } else if ($tablename == "supplierdetails") {
+                $supplierInsertQ->execute($supplierInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    echo $data[2] . " failed supplier insert.<br>" . print_r($errors, true) . " <br> " . print_r($supplierInsert, true) . "<br> $row <br><br>\n";
+                }
+            } else if ($tablename == "agency") {
+                $agencyInsertQ->execute($agencyInsert);
+                $errors = $conn->errorInfo();
+                if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+                    
+                } elseif ($errors[1] == 0) {
+                    $success++;
+                } else {
+                    echo $data[2] . " failed agency insert.<br>" . print_r($errors, true) . " <br> " . print_r($agencyInsert, true) . "<br> $row <br><br>\n";
+                }
+            }
+            flush();
+            //echo "<hr>\n";
+        }
+        $row++;
+    }
+    fclose($handle);
+    $contractNoticeInsertQ->closeCursor();
+    $supplierInsertQ->closeCursor();
+    $agencyInsertQ->closeCursor();
+ 
+    return $success;
+}
+
+$path = 'data/';
+if ($_REQUEST["fname"] == "") {
+    echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+    $dhandle = opendir($path);
+    // define an array to hold the files
+    $files = array();
+    if ($dhandle) {
+        // loop through all of the files
+        while (false !== ($fname = readdir($dhandle))) {
+            if (($fname != '.') && ($fname != '..')) {
+                echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
+            }
+        }
+    }
+} else {
+    $success = 0;
+    $fname = $_REQUEST["fname"];
+    echo " ============== $fname  ============== <br>";
+    flush();
+    $success+= processFile($path . $fname, "contractnotice");
+    $success+= processFile($path . $fname, "agency");
+    $success+= processFile($path . $fname, "supplier");
+    echo "<br> $success records successfully created";
+    
+    flush();
+       // run post import data processing
 // cn
-echo "link amend<br>";
-include ("linkAmendments.php");
-echo "update UNSPSC<br>";
-include ("updateUNSPSC.php");
+    echo "link amend<br>";
+    include ("linkAmendments.php");
+    echo "update UNSPSC<br>";
+    include ("updateUNSPSC.php");
 // agency
 //include ("setAgencyStatus.php");
 //include ("setAgencyURLABN.php");
-	return $success;
-}
-$path = 'data/';
-if ($_REQUEST["fname"] == "") {
-	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
-	$dhandle = opendir($path);
-	// define an array to hold the files
-	$files = array();
-	if ($dhandle) {
-		// loop through all of the files
-		while (false !== ($fname = readdir($dhandle))) {
-			if (($fname != '.') && ($fname != '..')) {
-				echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
-			}
-		}
-	}
-}
-else {
-	$success = 0;
-	$fname = $_REQUEST["fname"];
-	echo " ============== $fname  ============== <br>";
-	flush();
-	$success+= processFile($path . $fname, "contractnotice");
-	$success+= processFile($path . $fname, "agency");
-	$success+= processFile($path . $fname, "supplier");
-	echo "<br> $success records successfully created";
-	flush();
-}
-
+}
 ?>
file:a/admin/linkAmendments.php -> file:b/admin/linkAmendments.php 
--- a/admin/linkAmendments.php
+++ b/admin/linkAmendments.php
@@ -2,7 +2,7 @@
 include_once ("../lib/common.inc.php");
 $query = 'SELECT c."CNID",c."parentCN",p."childCN" FROM contractnotice as c  LEFT OUTER JOIN contractnotice as p on c."parentCN" = p."CNID"
 WHERE
-c."parentCN" > 0 AND p."childCN" IS NULL ';
+c."parentCN" IS NOT NULL AND p."childCN" IS NULL ';
 $query = $conn->prepare($query);
 	$query->execute();
 		databaseError($conn->errorInfo());
@@ -28,9 +28,9 @@
       FROM contractnotice
       GROUP BY "parentCN" 
       HAVING COUNT(*) > 1 
-      AND "parentCN" != 0
+      AND "parentCN" IS NOT NULL
 )
-AND "childCN" = 0
+AND "childCN" IS NULL
 GROUP BY "parentCN" having count(*) > 1';
 $query = $conn->prepare($query);
 	$query->execute();
file:a/admin/partialdata/scraper.php (deleted) 
--- a/admin/partialdata/scraper.php
+++ /dev/null
@@ -1,74 +1,1 @@
-<?php
-date_default_timezone_set('Australia/Melbourne');
-$split = false;
-function format_bytes($size) {
-    $units = array(' B', ' KB', ' MB', ' GB', ' TB');
-    for ($i = 0; $size >= 1024 && $i < 4; $i++) $size /= 1024;
-    return round($size, 2).$units[$i];
-}
 
-$days = 4;
-if (isset($_REQUEST['days'])) $days = $_REQUEST['days'];
-$startDate = strtotime("05-Jun-2008");
-if (isset($_REQUEST['startDate'])) $startDate = $_REQUEST['startDate'];
-
-function getFile($startDate, $days, $minVal, $maxVal) {
-global $split;
-	$endDate = strtotime(date("Y-m-d", $startDate)." +".$days." days");
-$file = date("dMY",$startDate).'to'.date("dMY",$endDate).'val'.$minVal.'to'.$maxVal.'.xls';
-echo "Fetching $file ($days days) ($minVal < value < $maxVal )... ";
-$url = "https://www.tenders.gov.au/?event=public.advancedsearch.CNSONRedirect&type=cnEvent&atmType=archived%2Cclosed%2Cpublished%2Cproposed&agencyUUID=&agencyStatus=-1&portfolioUUID=&keyword=&KeywordTypeSearch=AllWord&CNID=&dateType=Publish+Date&dateStart=".date("d-M-Y",$startDate)."&dateEnd=".date("d-M-Y",$endDate)."&supplierName=&supplierABN=&valueFrom=".$minVal."&valueTo=".$maxVal."&ATMID=&AgencyRefId=&consultancy=&download=Download+results";
-echo "<!-- $url -->";
-$current = file_get_contents($url);
-if (strpos($current,"There are no results that match your selection.")> 0 ) { 
- echo "<font color=red>Empty file!</font><br>";
-}
-if (strpos($current,"Your search returned more than 1000 results.") === false) {
-	file_put_contents($file, $current);
-	echo "$file saved<br>";
-	echo format_bytes(filesize($file))."<br>";
-	echo '<a href="?startDate='.$endDate.'&days='.$days.'">Load next '.($days).' days </a><br>';
-		echo '<a href="?startDate='.$endDate.'&days='.($days*2).'">Load next '.($days*2).' days </a><br>';
-	echo '<a href="?startDate='.$endDate.'&days='.$days.'&split=yes">Load next '.($days).' days with split</a><br>';
-	flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Success so fetching next $days... <br>";
-getFile($endDate, $days, "" , "");
-}
-	return true;
-} else  {
-	echo "<font color=red>Too many records!</font><br>";
-	echo '<a href="?startDate='.$startDate.'&days='.floor($days/2).'">Load '.($days/2).' days instead?</a><br>';
-		echo '<a href="?startDate='.$startDate.'&days='.$days.'&split=yes">Split instead?</a><br>';
-	flush();
-if (!isset($_REQUEST['split']) && !$split) {
-echo "Failure so splitting ... <br>";
- doSplit($startDate, $days);
-}
-	return false;
-}
-}
-function doSplit($startDate, $days) {
-global $split;
-$split = true;
-set_time_limit(20);
-getFile($startDate, $days, 0, 12000);
-getFile($startDate, $days, 12000, 16000);
- getFile($startDate, $days, 16000, 20000);
- getFile($startDate, $days, 20000, 30000);
- getFile($startDate, $days, 30000, 40000);
-// getFile($startDate, $days, 40000, 80000);
- getFile($startDate, $days, 40000, 60000);
- getFile($startDate, $days, 60000, 80000);
-// getFile($startDate, $days, 80000, 300000);
- getFile($startDate, $days, 80000, 150000);
- getFile($startDate, $days, 150000, 300000);
- getFile($startDate, $days, 300000, 999999999);
-}
-if (isset($_REQUEST['split'])) {
-	doSplit($startDate, $days);
-} else {
-	getFile($startDate, $days, "" , "");
-}
-?>
-
file:a/admin/updateUNSPSC.php -> file:b/admin/updateUNSPSC.php 
--- a/admin/updateUNSPSC.php
+++ b/admin/updateUNSPSC.php
@@ -19,12 +19,33 @@
     $unspsc[$armor] = $row['UNSPSC'];

     $erre = str_replace("er", "re", $row['Title']);

     $unspsc[$erre] = $row['UNSPSC'];

+    $center = str_replace("center", "centre", $row['Title']);

+    $unspsc[$center] = $row['UNSPSC'];

+      $accessory = str_replace("accesor", "accessor", $row['Title']);

+    $unspsc[$accessory] = $row['UNSPSC'];

     $lyslyz = str_replace("lyz", "lys", $row['Title']);

     $unspsc[$lyslyz] = $row['UNSPSC'];

+        $tire = str_replace("ire", "yre", $row['Title']);

+    $unspsc[$tire] = $row['UNSPSC'];

+    

+        $pe = str_replace("pe", "pae", $row['Title']);

+    $unspsc[$pe] = $row['UNSPSC'];

+            $ane = str_replace("ane", "anae", $row['Title']);

+    $unspsc[$ane] = $row['UNSPSC'];

+        $airo = str_replace("airplane", "aeroplane", $row['Title']);

+    $unspsc[$airo] = $row['UNSPSC'];

     // some divergence from standard

+      $forensicit = str_replace("Information technology consultation services", "Forensic IT Services", $row['Title']);

+        $unspsc[$forensicit] = $row['UNSPSC'];

+        $powercable = str_replace( "Power cable", "Power cable installation and supply", $row['Title']);

+    $unspsc[$powercable] = $row['UNSPSC'];

     $tobacco = str_replace("Food Beverage and Tobacco Products", "Food and Beverage Products", $row['Title']);

     $unspsc[$tobacco] = $row['UNSPSC'];

     $architect = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);

+    $unspsc[$architect] = $row['UNSPSC'];

+        $powercable = str_replace("Power cable", "Power cable installation and supply", $row['Title']);

+    $unspsc[$powercable] = $row['UNSPSC'];

+        $forensicIT = str_replace("Building and Construction and Maintenance Services", "Architectural services", $row['Title']);

     $unspsc[$architect] = $row['UNSPSC'];

     // some just plain wrong

     $noOilRigs = str_replace("Building and Construction and Maintenance Services", "Management and provision of all facilities engineering modification and maintenance services for a site or platform", $row['Title']);
file:a/australian_federal_government_contract_spending.json -> file:b/australian_federal_government_contract_spending.json 
--- a/australian_federal_government_contract_spending.json
+++ b/australian_federal_government_contract_spending.json
@@ -1,125 +1,164 @@
 {
-   "dataset":{
-      "name":"australian_federal_government_contract_spending",
-      "label":"Australian Federal Government Contract Spending",
-      "description":"Spending by Australian Federal Government agencies on goods and services from 2007 onwards.",
-      "currency":"AUD",
-      "unique_keys":[
-         "id"
-      ],
-      "temporal_granularity":"year"
-   },
-   "mapping":{
-  "category": {
-      "fields": [ {
-         "column": "category", 
-        "datatype": "string", 
-          "name": "label"
-          } ],
-
-    "type": "classifier", 
-    "description": "", 
-    "taxonomy": "unspsc",
-    "label": "Contract Goods/Services Category"
+  "dataset": {
+    "languages": [
+      "en"
+    ], 
+    "currency": "AUD", 
+    "name": "australian_federal_government_contract_spending", 
+    "territories": [], 
+    "default_time": null, 
+    "description": "Spending by Australian Federal Government agencies on goods and services from 2007 onwards.", 
+    "schema_version": "2011-12-07", 
+    "label": "Australian Federal Government Contract Spending"
   }, 
-      "from":{
-         "fields":[
-            {
-               "column":"agencyABN",
-               "datatype":"string",
-               "name":"id"
-            },
-            {
-               "column":"agencyName",
-               "datatype":"string",
-               "name":"label"
-            }
-         ],
-         "type":&quo