Link amendments made in new numbering scheme
[contractdashboard.git] / admin / partialdata / import.php
blob:a/admin/partialdata/import.php -> blob:b/admin/partialdata/import.php
--- a/admin/partialdata/import.php
+++ b/admin/partialdata/import.php
@@ -1,1 +1,146 @@
+<?php
+include_once ("../lib/common.inc.php");
+function processFile($fpath, $tablename)
+{
+	global $conn;
+	$row = 1;
+	$handle = fopen($fpath, "r");
+	//"t" mode string translates windows line breaks to unix
+	$datamapping0507 = array(
+		"Agency" => "agencyName",
+		"CN ID" => "CNID",
+		"Publish Date" => "publishDate",
+		"Contract Start Date" => "contractStart",
+		"Contract End Date" => "contractEnd",
+		"Value (AUD)" => "value",
+		"Title" => "description",
+		"Category" => "category",
+		"ATM ID" => "atmID",
+		"LastUpdated" => "",
+		"" => ""
+	);
+	$headers;
+	$contractNoticeFields = array(
+		"importFile",
+		"CNID",
+		"description",
+		"agencyName",
+		"publishDate",
+		"category",
+		"contractStart",
+		"contractEnd",
+		"value",
+		"atmID",
+		"supplierName",
+		"LastUpdated"
+	);
+	if ($tablename == "contractnotice") {
+		$contractNoticeInsertQ = 'INSERT INTO contractnotice ("' . implode('" , "', $contractNoticeFields) . '") VALUES ( ';
+		foreach ($contractNoticeFields as $key => $f) {
+			$contractNoticeInsertQ.= ($key == 0 ? "" : ", ") . "?";
+		}
+		$contractNoticeInsertQ.= ");";
+		$contractNoticeInsertQ = $conn->prepare($contractNoticeInsertQ);
+	}
+	
+	while (($data = fgetcsv($handle, 1000, "\t")) !== false) {
+		$num = count($data);
+		if ($row == 3) {
+			$headers = $data;
+		}
+		elseif ($row > 3) {
+			if ($num > count($datamapping0507)) {
+				die("<font color=red>Error in data import; data mapping fields out of bounds or changed</font><br>" . $fname . print_r($data));
+			}
+			$contractNoticeInsert = Array();
+			$contractNoticeInsert[] = $fpath;
+			$keys = array_keys($datamapping0507);
+			for ($c = 0; $c < $num; $c++) {
+				$data[$c] = trim($data[$c], "=");
+				$data[$c] = trim($data[$c], "\"");
+				if ($tablename == "contractnotice") {
+					if (in_array(($datamapping0507[$headers[$c]]) , $contractNoticeFields)) {
+						if (($datamapping0507[$headers[$c]]) == "parentCN" || ($datamapping0507[$headers[$c]]) == "CNID") {
+							$data[$c] = substr($data[$c], 2); // take off the "CN" prefix
+							$data[$c] = str_replace("-A", "00", $data[$c]); // make amendments really big numbers
+							if (!is_numeric($data[$c]) && $data[$c] != "") die($data[$c] . " is not numeric");
+							if ($data[$c] > 0) {
+								$contractNoticeInsert[] = $data[$c];
+							}
+							else {
+								$contractNoticeInsert[] = 0;
+							}
+						}
+						elseif (($datamapping0507[$headers[$c]]) == "supplierABN") {
+							if ($data[$c] > 0) {
+								$contractNoticeInsert[] = $data[$c];
+							}
+							else {
+								$contractNoticeInsert[] = null;
+							}
+						}
+						elseif (($datamapping0507[$headers[$c]]) == "amendDate" || ($datamapping0507[$headers[$c]]) == "publishDate" || ($datamapping0507[$headers[$c]]) == "contractStart" || ($datamapping0507[$headers[$c]]) == "contractEnd") {
+							$contractNoticeInsert[] = date('Y-m-d H:i:s', strtotime($data[$c]));
+						}
+						else {
+							if (strstr("\" =", $data[$c] > 0)) {
+								die("Invalid Description field" . $contractNoticeInsert);
+							}
+$colvalue =  preg_replace( '/[^[:print:]]/', '',utf8_encode( $data[$c]));
 
+							$contractNoticeInsert[] = $colvalue;
+						}
+					}
+				}
+			}
+			flush();
+			if ($tablename == "contractnotice") {
+				$contractNoticeInsertQ->execute($contractNoticeInsert);
+				$errors = $conn->errorInfo();
+				if ($errors[1] == 7 && strpos($errors[2], "duplicate key")) {
+				}
+				elseif ($errors[1] == 0) {
+					$success++;
+				}
+				else {
+					foreach ($contractNoticeFields as $key => $cnf) {
+						echo var_dump($contractNoticeInsert[$key]) . $cnf . "<br>";
+					}
+					echo $data[2] . " failed CN insert.<br>" . print_r($errors, true) . "<br> row $row <br><br>\n";
+				}
+			}
+			flush();
+			//echo "<hr>\n";
+			
+		}
+		$row++;
+	}
+	fclose($handle);
+	return $success;
+}
+$path = './';
+if ($_REQUEST["fname"] == "") {
+	echo "Get files from: https://www.tenders.gov.au/?event=public.reports.list<br>";
+	$dhandle = opendir($path);
+	// define an array to hold the files
+	$files = array();
+	if ($dhandle) {
+		// loop through all of the files
+		while (false !== ($fname = readdir($dhandle))) {
+			if (($fname != '.') && ($fname != '..')) {
+				echo "<a href=\"import.php?fname=$fname\">$fname</a>&nbsp;" . filesize($path . $fname) . "&nbsp;" . date("c", filemtime($path . $fname)) . "<br/>";
+			}
+		}
+	}
+}
+else {
+	$success = 0;
+	$fname = $_REQUEST["fname"];
+	echo " ============== $fname  ============== <br>";
+	flush();
+	$success+= processFile($path . $fname, "contractnotice");
+	echo "<br> $success records successfully created";
+	flush();
+}
+?>
+