fix heuristics
--- a/heuristics/heuristics.inc.php
+++ b/heuristics/heuristics.inc.php
@@ -1,43 +1,47 @@
<?php
+
include_once ("../lib/common.inc.php");
$heuristics = Array();
//each heuristic adds self to description array
include ("dateHeuristics.php");
include ("historyHeuristics.php");
+
//include ("metadataHeuristics.php");
//include ("valueHeuristics.php");
-function runHeuristic($heuristicName, $cn)
-{
+function runHeuristic($heuristicName, $cn) {
global $conn;
- // check if already ran
- $query = "select count(*) from heuristic_results where heuristic_name = '$heuristicName' and \"CNID\" = '{$cn['CNID']}'";
- $result = $conn->query($query);
- databaseError($conn->errorInfo());
+ // check if already ran
+ $query = "select count(*) from heuristic_results where heuristic_name = '$heuristicName' and \"CNID\" = '{$cn['CNID']}'";
+ $result = $conn->query($query);
+ databaseError($conn->errorInfo());
$r = $result->fetch(PDO::FETCH_BOTH);
- if ($r[0] == 0) {
- // if not, run now
- $hresults = call_user_func($heuristicName, $cn);
- if (!isset($hresults["heuristic_value"]) || !isset($hresults["raw_value"]) || !isset($hresults["mean"]) || !isset($hresults["stddev"])) {
- print_r($hresults);
- die("Missing field in heurtistic $heuristicName result");
- }
- $query = "insert into heuristic_results values('$heuristicName',
+ if ($r[0] == 0) {
+ // if not, run now
+ $hresults = call_user_func($heuristicName, $cn);
+ if (!isset($hresults["heuristic_value"]) || !isset($hresults["raw_value"]) || !isset($hresults["mean"]) || !isset($hresults["stddev"])) {
+ print_r($hresults);
+ die("Missing field in heurtistic $heuristicName result");
+ }
+ $query = "insert into heuristic_results values('$heuristicName',
'{$hresults["heuristic_value"]}',
'{$hresults["raw_value"]}',
'{$hresults["mean"]}',
'{$hresults["stddev"]}',
'{$cn["CNID"]}',
NOW(),
- '{$cn["publishDate"]}',
- '{$cn["agencyABN"]}',
- '{$cn["supplierID"]}'
+ '{$cn["publishDate"]}',"
+ //."'{$cn["agencyABN"]}',
+ ."0,'{$cn["supplierID"]}'
)";
- // save value and cn data via sql
- $result = $conn->query($query);
- $errors = $conn->errorInfo();
- if ($errors[2] == "") echo "Saved $heuristicName for {$cn["CNID"]} <br>\n";
- elseif (strpos($errors[2] , "Duplicate entry") === false) echo $hresults . " failed insert.<br>" . print_r($errors,true) . " <br> $query <br><br>\n";
- }
+ // save value and cn data via sql
+ $result = $conn->query($query);
+ $errors = $conn->errorInfo();
+ if ($errors[2] == "")
+ echo "Saved $heuristicName for {$cn["CNID"]} <br>\n";
+ elseif (strpos($errors[2], "Duplicate entry") === false)
+ echo $hresults . " failed insert.<br>" . print_r($errors, true) . " <br> $query <br><br>\n";
+ }
}
+
?>
--- a/heuristics/historyHeuristics.php
+++ b/heuristics/historyHeuristics.php
@@ -20,9 +20,9 @@
$agencyTransactions = Array();
function getAgencyTransactions($agencyName)
{
- global $agencyTransactions;
+ global $agencyTransactions,$conn;
if (!$agencyTransactions[$agencyName]) {
- $query = 'select count(*) from contractnotice where agencyName = "' . $agencyName . '"';
+ $query = 'select count(*) from contractnotice where "agencyName" = \'' . $agencyName . '"\'';
$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);
$agencyTransactions[$agencyName] = $r[0];
@@ -49,9 +49,9 @@
}
function getStatsAgencyTransactions()
{
- global $averageAgencyTransactions, $stddevAgencyTransactions;
- $query = "select avg(count), STDDEV(count) from (select count(*) as count
- from contractnotice group by agencyName) as a;";
+ global $averageAgencyTransactions, $stddevAgencyTransactions,$conn;
+ $query = 'select avg(count), STDDEV(count) from (select count(*) as count
+ from contractnotice group by "agencyName") as a;';
$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);
$averageAgencyTransactions = $r[0];
@@ -78,10 +78,10 @@
$supplierTransactions = Array();
function getSupplierTransactions($supplierName, $supplierABN)
{
- global $supplierTransactions;
+ global $supplierTransactions,$conn;
if ($supplierABN != 0 && $supplierABN != "") {
if (!$supplierTransactions[$supplierABN]) {
- $query = 'select count(*) from contractnotice where supplierABN = "' . $supplierABN . '"';
+ $query = "select count(*) from contractnotice where \"supplierABN\" = '" . $supplierABN . "'";
$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);
$supplierTransactions[$supplierABN] = $r[0];
@@ -89,7 +89,8 @@
return $supplierTransactions[$supplierABN];
}
if (!$supplierTransactions[$supplierName]) {
- $query = 'select count(*) from contractnotice where supplierName = "' . $supplierName . '"';
+ $query = "select count(*) from contractnotice where \"supplierName\" = '" . $supplierName . "'";
+
$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);
$supplierTransactions[$supplierName] = $r[0];
@@ -116,8 +117,10 @@
}
function getStatsSupplierTransactions()
{
- global $averageSupplierTransactions, $stddevSupplierTransactions;
- $query = 'select avg(count), stddev(count) from (select IF(supplierABN != "",supplierABN,supplierName) as supplierID, count(*) as count from contractnotice group by supplierID) as a;';
+ global $averageSupplierTransactions, $stddevSupplierTransactions,$conn;
+ $query = 'select avg(count), stddev(count) from (
+ select (case when "supplierABN" != 0 THEN "supplierABN"::text ELSE "supplierName" END) as "supplierID",
+ count(*) as count from contractnotice group by "supplierID") as a;';
$result = $conn->query($query);
$r = $result->fetch(PDO::FETCH_BOTH);
$averageSupplierTransactions = $r[0];
--- a/heuristics/runHeuristics.php
+++ b/heuristics/runHeuristics.php
@@ -5,10 +5,11 @@
auth();
include_once("heuristics.inc.php");
-$query = 'SELECT *, agency.abn as "agencyABN", case when "supplierABN" != 0 then "supplierABN"::text else "supplierName" end as "supplierID"
-FROM contractnotice JOIN agency ON contractnotice."agencyName"=agency."agencyName"
+// agency table missing JOIN agency ON contractnotice."agencyName"
+$query = 'SELECT *, case when "supplierABN" != 0 then "supplierABN"::text else "supplierName" end as "supplierID"
+FROM contractnotice
WHERE DATE("importDate") = (select * from (SELECT DATE("importDate")
-FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) limit 10';
+FROM contractnotice ORDER BY "importDate" DESC limit 1) alias) limit 100';
$query = $conn->prepare($query);
$query->execute();
databaseError($conn->errorInfo());
--- a/heuristics/valueHeuristics.php
+++ b/heuristics/valueHeuristics.php
@@ -1,58 +1,61 @@
-<?php
- /* - large contract value
- - standard dev from mean/median
- - percent of total contracts for supplier/agency*/
- $heuristics["VALUE_LARGE_CONTRACT_OVERALL"] = Array(
- "description" => "unusual value for time of year");
-function METADATA_DUPLICATED_DESCRIPTION($cn)
-{
- $averageContractPeriod = getAverageContractPeriod();
- $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
- $days = intval($diff / 24);
- return ($days > 45 ? 1 : 0);
+<?php
+
+/* - large contract value
+ - standard dev from mean/median
+ - percent of total contracts for supplier/agency */
+$heuristics["VALUE_LARGE_CONTRACT_OVERALL"] = Array(
+ "description" => "unusual value for time of year");
+
+function VALUE_LARGE_CONTRACT_OVERALL($cn) {
+ $averageContractPeriod = getAverageContractPeriod();
+ $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
+ $days = intval($diff / 24);
+ return ($days > 45 ? 1 : 0);
}
- /* - peculiar value
- - Just under 80k, amplified if other contracts with same supplier are just under
- */
- $heuristics["VALUE_NEAR_THRESHOLD"] = Array(
- "description" => "unusual value for time of year");
- /*
- - unusual variation amount - absolute value; large reductions as well as large increases
- */
- $heuristics["VALUE_LARGE_VARIATION"] = Array(
- "description" => "unusual value for time of year");
-function METADATA_DUPLICATED_DESCRIPTION($cn)
-{
- $averageContractPeriod = getAverageContractPeriod();
- $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
- $days = intval($diff / 24);
- return ($days > 45 ? 1 : 0);
+/* - peculiar value
+ - Just under 80k, amplified if other contracts with same supplier are just under
+ */
+$heuristics["VALUE_NEAR_THRESHOLD"] = Array(
+ "description" => "unusual value for time of year");
+/*
+ - unusual variation amount - absolute value; large reductions as well as large increases
+ */
+$heuristics["VALUE_LARGE_VARIATION"] = Array(
+ "description" => "unusual value for time of year");
+
+function VALUE_LARGE_VARIATION($cn) {
+ $averageContractPeriod = getAverageContractPeriod();
+ $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
+ $days = intval($diff / 24);
+ return ($days > 45 ? 1 : 0);
}
/* - unusual value for time of year
- - compare to all other records in last 2 weeks
- - ie. many large contracts in june so takes more to standout*/
+ - compare to all other records in last 2 weeks
+ - ie. many large contracts in june so takes more to standout */
$heuristics["VALUE_HIGH_FOR_MONTH"] = Array(
- "description" => "unusual value for time of year"
+ "description" => "unusual value for time of year"
);
-function VALUE_HIGH_FOR_MONTH($cn, $monthAsInt)
-{
- $averageContractPeriod = getAverageContractPeriod();
- $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
- $days = intval($diff / 24);
- return ($days > 45 ? 1 : 0);
+
+function VALUE_HIGH_FOR_MONTH($cn, $monthAsInt) {
+ $averageContractPeriod = getAverageContractPeriod();
+ $diff = strtotime($cn['contractStart']) - strtotime($cn['publishDate']);
+ $days = intval($diff / 24);
+ return ($days > 45 ? 1 : 0);
}
+
$monthlyValueAverage = Array();
-function getAgencyTransactions($agencyName)
-{
- global $agencyTransactions;
- if (!$agencyTransactions[$agencyName]) {
- $query = 'select count(*) from contractnotice where agencyName = "' . $agencyName . '"';
- $result = $conn->query($query);
- $r = $result->fetch(PDO::FETCH_BOTH);
- $agencyTransactions[$agencyName] = $r[0];
- }
- return $agencyTransactions[$agencyName];
+
+function getAgencyTransactions($agencyName) {
+ global $agencyTransactions;
+ if (!$agencyTransactions[$agencyName]) {
+ $query = 'select count(*) from contractnotice where agencyName = "' . $agencyName . '"';
+ $result = $conn->query($query);
+ $r = $result->fetch(PDO::FETCH_BOTH);
+ $agencyTransactions[$agencyName] = $r[0];
+ }
+ return $agencyTransactions[$agencyName];
}
+
?>
--- a/heuristics/viewHeuristicsDistribution.php
+++ b/heuristics/viewHeuristicsDistribution.php
@@ -6,7 +6,7 @@
// spread of values
select floor(sum) as val,count(*) from (SELECT sum(heuristic_value)
-as sum FROM heuristic_results group by CNID) as a group by val*/
+as sum FROM heuristic_results group by "CNID") as a group by val*/
$series = Array();
@@ -17,4 +17,17 @@
$series[$r["heuristic_name"]][$r["val"]] = $r[2];
}
+ $labels = Array();
+ foreach ($series as $value) {
+ $labels = $labels+array_keys($value);
+ }
+ $labels = Array(0,1,2,3,4,5);
+foreach ($series as $seriesName => $seriesEntry) {
+ $data;
+ foreach ($labels as $label) {
+ $data[$label] = ($seriesEntry[$label] ? $seriesEntry[$label] : 0);
+ }
+}
+print_r($labels);
+print_r($data);
?>
--- a/lib/common.inc.php
+++ b/lib/common.inc.php
@@ -11,7 +11,7 @@
}
define('ROOT', pathinfo(__FILE__, PATHINFO_DIRNAME));
-if (strstr($_SERVER['PHP_SELF'], "labs/") || strstr($_SERVER['PHP_SELF'], "admin/")) {
+if (strstr($_SERVER['PHP_SELF'], "labs/") || strstr($_SERVER['PHP_SELF'], "admin/") || strstr($_SERVER['PHP_SELF'], "heuristics/")) {
$basePath = "../";
}
require $basePath."lib/amon-php/amon.php";