gitphp 0.2.9.1 :: disclosr.git/commitdiff

add RTK import

Former-commit-id: a6b6a0a350cf467915ec023c9e41421df3fcb83d

2 files changed: (show all)
admin/importRTKbodies.php (new)
alaveteli/exportAgencies.csv.php

file:b/admin/importRTKbodies.php (new)

	<?php

	require_once '../include/common.inc.php';

	$db = $server->get_db('disclosr-agencies');
	$rows = $db->get_view("app", "byName")->rows;
	$nametoid = Array();
	$accounts = Array();
	foreach ($rows as $row) {
	$nametoid[trim($row->key)] = $row->value;
	}

	function extractCSVAccounts($url, $nameField, $accountField, $filter) {
	global $accounts, $nametoid;
	$request = Requests::get($url);
	echo $url;
	$Data = str_getcsv($request->body, "\n"); //parse the rows
	$headers = Array();
	foreach ($Data as $num => $line) {
	$Row = str_getcsv($line, ",");
	if ($num == 0) {
	$headers = $Row;
	print_r($headers);
	} else {
	if (isset($Row[array_search($nameField, $headers)])) {
	$agencyName = $Row[array_search($nameField, $headers)];
	if (!in_array(trim($agencyName), array_keys($nametoid))) {
	echo "$agencyName missing" . PHP_EOL;
	} else {
	echo $Row[array_search($nameField, $headers)] . PHP_EOL;
	$accounts[$nametoid[trim($agencyName)]]["rtkURLs"][$agencyName] = 'http://www.righttoknow.org.au/body/'.$Row[array_search($accountField, $headers)];
	}
	} else {
	echo "error finding any agency" . $line . PHP_EOL;
	}
	}
	}
	}

	extractCSVAccounts("http://www.righttoknow.org.au/body/all-authorities.csv","Agency","URL name");
	print_r($accounts);
	/* foreach ($accounts as $id => $accountTypes) {
	echo $id . "<br>" . PHP_EOL;
	$doc = object_to_array($db->get($id));
	// print_r($doc);

	foreach ($accountTypes as $accountType => $accounts) {
	if (!isset($doc["has" . $accountType]) \|\| !is_array($doc["has" . $accountType])) {
	$doc["has" . $accountType] = Array();
	}
	$doc["has" . $accountType] = array_unique(array_merge($doc["has" . $accountType], $accounts));
	}
	$db->save($doc);
	}*/
	?>

file:a/alaveteli/exportAgencies.csv.php -> file:b/alaveteli/exportAgencies.csv.php

<?php	<?php

include_once("../include/common.inc.php");	include_once("../include/common.inc.php");

function shortName($name) {	function shortName($name) {
$name = trim($name);	$name = trim($name);
if (strstr($name,"Minister ") \|\| strstr($name,"Treasurer") \|\| strstr($name,"Parliamentary Secretary")) {	if (strstr($name,"Minister ") \|\| strstr($name,"Treasurer") \|\| strstr($name,"Parliamentary Secretary")) {
$badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");	$badWords = Array ("Assisting the Prime Minister on","Assisting on"," the "," of "," for "," on "," and "," to ",","," ","'","`");
return str_replace($badWords,"",$name);	return str_replace($badWords,"",$name);
}	}

else {	else {
$out = Array();	$out = Array();
preg_match_all('/[A-Z]/', $name, $out);	preg_match_all('/[A-Z]/', $name, $out);
return implode("", $out[0]);	return implode("", $out[0]);
}	}
}	}

setlocale(LC_CTYPE, 'C');	setlocale(LC_CTYPE, 'C');

$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");	$headers = Array("#id", "name", "request_email", "short_name", "notes", "publication_scheme", "home_page", "tag_string");

$db = $server->get_db('disclosr-agencies');	$db = $server->get_db('disclosr-agencies');

$tag = Array();	$tag = Array();
try {	try {
$rows = $db->get_view("app", "byDeptStateName", null, true)->rows;	$rows = $db->get_view("app", "byDeptStateName", null, true)->rows;
//print_r($rows);	//print_r($rows);
foreach ($rows as $row) {	foreach ($rows as $row) {
$tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));	$tag[$row->id] = phrase_to_tag(dept_to_portfolio($row->key));
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
die();	die();
}	}

$foiEmail = Array();	$foiEmail = Array();
try {	try {
$rows = $db->get_view("app", "foiEmails", null, true)->rows;	$rows = $db->get_view("app", "foiEmails", null, true)->rows;
//print_r($rows);	//print_r($rows);
foreach ($rows as $row) {	foreach ($rows as $row) {
$foiEmail[$row->key] = $row->value;	$foiEmail[$row->key] = $row->value;
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
die();	die();
}	}

$fp = fopen('php://output', 'w');	$fp = fopen('php://output', 'w');
if ($fp && $db) {	if ($fp && $db) {
header('Content-Type: text/csv; charset=utf-8');	header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');	header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"');
header('Pragma: no-cache');	header('Pragma: no-cache');
header('Expires: 0');	header('Expires: 0');
fputcsv($fp, $headers);	fputcsv($fp, $headers);
try {	try {
$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;	$agencies = $db->get_view("app", "byCanonicalName", null, true)->rows;
//print_r($rows);	//print_r($rows);
foreach ($agencies as $agency) {	foreach ($agencies as $agency) {
// print_r($agency);	// print_r($agency);

if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {	if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) {
$row = Array();	$row = Array();
$row["#id"] = $agency->id;	$row["#id"] = $agency->id;
$row["name"] = trim($agency->value->name);	$row["name"] = trim($agency->value->name);
if (isset($agency->value->foiEmail)) {	if (isset($agency->value->foiEmail)) {
$row["request_email"] = $agency->value->foiEmail;	$row["request_email"] = $agency->value->foiEmail;
} else {	} else {
if ($agency->value->orgType == "FMA-DepartmentOfState") {	if ($agency->value->orgType == "FMA-DepartmentOfState") {
$row["request_email"] = "foi@" . GetDomain($agency->value->website);	$row["request_email"] = "foi@" . GetDomain($agency->value->website);
} else {	} else {
$row["request_email"] = $foiEmail[$agency->value->parentOrg];	$row["request_email"] = $foiEmail[$agency->value->parentOrg];
}	}
}	}
if (isset($agency->value->shortName)) {	if (isset($agency->value->shortName)) {
$row["short_name"] = $agency->value->shortName;	$row["short_name"] = $agency->value->shortName;
} else {	} else {
$row["short_name"] = shortName($agency->value->name);	$row["short_name"] = shortName($agency->value->name);
}	}
$row["notes"] = "";	$row["notes"] = "";
$row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");	$row["publication_scheme"] = (isset($agency->value->infoPublicationSchemeURL) ? $agency->value->infoPublicationSchemeURL : "");
$row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");	$row["home_page"] = (isset($agency->value->website) ? $agency->value->website : "");
if ($agency->value->orgType == "FMA-DepartmentOfState") {	if ($agency->value->orgType == "FMA-DepartmentOfState") {
$row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;	$row["tag_string"] = $tag[$agency->value->_id] . " " . $agency->value->orgType;
} else {	} else {
$row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;	$row["tag_string"] = $tag[$agency->value->parentOrg] . " " . $agency->value->orgType;
}	}

fputcsv($fp, array_values($row));	fputcsv($fp, array_values($row));

if (isset($agency->value->foiBodies)) {	if (isset($agency->value->foiBodies)) {
foreach ($agency->value->foiBodies as $foiBody) {	foreach ($agency->value->foiBodies as $foiBody) {
$row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);	$row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$foiBody);
$row["short_name"] = shortName($foiBody);	$row["short_name"] = shortName($foiBody);
fputcsv($fp, array_values($row));	fputcsv($fp, array_values($row));
}	}
}	}
	if (isset($agency->value->positions)) {
	foreach ($agency->value->positions as $position) {
	$row['name'] = iconv("UTF-8", "ASCII//TRANSLIT",$position);
	$row["short_name"] = shortName($position);
	fputcsv($fp, array_values($row));
	}
	}
}	}
}	}
} catch (SetteeRestClientException $e) {	} catch (SetteeRestClientException $e) {
setteErrorHandler($e);	setteErrorHandler($e);
}	}

die;	die;
}	}
?>	?>