New db import method
New db import method

<?php <?php
date_default_timezone_set('Australia/ACT'); date_default_timezone_set('Australia/ACT');
$debugOkay = Array( $debugOkay = Array(
"session", "session",
"json", "json",
"phperror", "phperror",
"awsotp", "awsotp",
//"squallotp", //"squallotp",
//"vanilleotp", //"vanilleotp",
"database", "database",
"other" "other"
); );
$cloudmadeAPIkey = "daa03470bb8740298d4b10e3f03d63e6"; $cloudmadeAPIkey = "daa03470bb8740298d4b10e3f03d63e6";
$googleMapsAPIkey = "ABQIAAAA95XYXN0cki3Yj_Sb71CFvBTPaLd08ONybQDjcH_VdYtHHLgZvRTw2INzI_m17_IoOUqH3RNNmlTk1Q"; $googleMapsAPIkey = "ABQIAAAA95XYXN0cki3Yj_Sb71CFvBTPaLd08ONybQDjcH_VdYtHHLgZvRTw2INzI_m17_IoOUqH3RNNmlTk1Q";
$otpAPIurl = 'http://localhost:8080/opentripplanner-api-webapp/'; $otpAPIurl = 'http://localhost:8080/opentripplanner-api-webapp/';
if (isDebug("awsotp") || php_uname('n') == "maxious.xen.prgmr.com") { if (isDebug("awsotp") || php_uname('n') == "maxious.xen.prgmr.com") {
$otpAPIurl = 'http://bus-main.lambdacomplex.org:8080/opentripplanner-api-webapp/'; $otpAPIurl = 'http://bus-main.lambdacomplex.org:8080/opentripplanner-api-webapp/';
} }
if (isDebug("dotcloudotp") || php_uname('n') == "actbus-www") { if (isDebug("dotcloudotp") || php_uname('n') == "actbus-www") {
$otpAPIurl = 'http://otp.actbus.dotcloud.com/opentripplanner-api-webapp/'; $otpAPIurl = 'http://otp.actbus.dotcloud.com/opentripplanner-api-webapp/';
} }
if (isDebug("squallotp")) { if (isDebug("squallotp")) {
$otpAPIurl = 'http://10.0.1.108:5080/opentripplanner-api-webapp/'; $otpAPIurl = 'http://10.0.1.108:5080/opentripplanner-api-webapp/';
} }
if (isDebug("vanilleotp")) { if (isDebug("vanilleotp")) {
$otpAPIurl = 'http://10.0.1.135:8080/opentripplanner-api-webapp/'; $otpAPIurl = 'http://10.0.1.135:8080/opentripplanner-api-webapp/';
} }
if (isDebug("phperror")) error_reporting(E_ALL ^ E_NOTICE); if (isDebug("phperror")) error_reporting(E_ALL ^ E_NOTICE);
$labsPath = ""; $labsPath = "";
if (strstr($_SERVER['PHP_SELF'],"labs")) $labsPath = "../"; if (strstr($_SERVER['PHP_SELF'],"labs")) $labsPath = "../";
   
function isDebugServer() function isDebugServer()
{ {
return isset($_SERVER['SERVER_NAME']) && ( $_SERVER['SERVER_NAME'] == "10.0.1.154" || $_SERVER['SERVER_NAME'] == "10.1.0.4" || $_SERVER['SERVER_NAME'] == return php_sapi_name() == "cli" || isset($_SERVER['SERVER_NAME']) && ( $_SERVER['SERVER_NAME'] == "10.0.1.154" || $_SERVER['SERVER_NAME'] == "10.1.0.4" || $_SERVER['SERVER_NAME'] ==
"localhost" || $_SERVER['SERVER_NAME'] == "127.0.0.1") ; "localhost" || $_SERVER['SERVER_NAME'] == "127.0.0.1") ;
} }
   
include_once ("common-geo.inc.php"); include_once ("common-geo.inc.php");
include_once ("common-net.inc.php"); include_once ("common-net.inc.php");
include_once ("common-transit.inc.php"); include_once ("common-transit.inc.php");
include_once ("common-db.inc.php"); include_once ("common-db.inc.php");
   
include_once ("common-request.inc.php"); include_once ("common-request.inc.php");
include_once ("common-session.inc.php"); include_once ("common-session.inc.php");
include_once ("common-template.inc.php"); include_once ("common-template.inc.php");
   
   
function isAnalyticsOn() function isAnalyticsOn()
{ {
return !isDebugServer(); return !isDebugServer();
} }
function isDebug($debugReason = "other") function isDebug($debugReason = "other")
{ {
global $debugOkay; global $debugOkay;
return in_array($debugReason, $debugOkay, false) && isDebugServer(); return in_array($debugReason, $debugOkay, false) && isDebugServer();
} }
function debug($msg, $debugReason = "other") function debug($msg, $debugReason = "other")
{ {
if (isDebug($debugReason)) echo "\n<!-- " . date(DATE_RFC822) . "\n $msg -->\n"; if (isDebug($debugReason)) echo "\n<!-- " . date(DATE_RFC822) . "\n $msg -->\n";
} }
function isJQueryMobileDevice() function isJQueryMobileDevice()
{ {
// http://forum.jquery.com/topic/what-is-the-best-way-to-detect-all-useragents-which-can-handle-jquery-mobile#14737000002087897 // http://forum.jquery.com/topic/what-is-the-best-way-to-detect-all-useragents-which-can-handle-jquery-mobile#14737000002087897
$user_agent = $_SERVER['HTTP_USER_AGENT']; $user_agent = $_SERVER['HTTP_USER_AGENT'];
return preg_match('/iphone/i', $user_agent) || preg_match('/android/i', $user_agent) || preg_match('/webos/i', $user_agent) || preg_match('/ios/i', $user_agent) || preg_match('/bada/i', $user_agent) || preg_match('/maemo/i', $user_agent) || preg_match('/meego/i', $user_agent) || preg_match('/fennec/i', $user_agent) || (preg_match('/symbian/i', $user_agent) && preg_match('/s60/i', $user_agent) && $browser['majorver'] >= 5) || (preg_match('/symbian/i', $user_agent) && preg_match('/platform/i', $user_agent) && $browser['majorver'] >= 3) || (preg_match('/blackberry/i', $user_agent) && $browser['majorver'] >= 5) || (preg_match('/opera mobile/i', $user_agent) && $browser['majorver'] >= 10) || (preg_match('/opera mini/i', $user_agent) && $browser['majorver'] >= 5); return preg_match('/iphone/i', $user_agent) || preg_match('/android/i', $user_agent) || preg_match('/webos/i', $user_agent) || preg_match('/ios/i', $user_agent) || preg_match('/bada/i', $user_agent) || preg_match('/maemo/i', $user_agent) || preg_match('/meego/i', $user_agent) || preg_match('/fennec/i', $user_agent) || (preg_match('/symbian/i', $user_agent) && preg_match('/s60/i', $user_agent) && $browser['majorver'] >= 5) || (preg_match('/symbian/i', $user_agent) && preg_match('/platform/i', $user_agent) && $browser['majorver'] >= 3) || (preg_match('/blackberry/i', $user_agent) && $browser['majorver'] >= 5) || (preg_match('/opera mobile/i', $user_agent) && $browser['majorver'] >= 10) || (preg_match('/opera mini/i', $user_agent) && $browser['majorver'] >= 5);
} }
function isFastDevice() function isFastDevice()
{ {
$ua = $_SERVER['HTTP_USER_AGENT']; $ua = $_SERVER['HTTP_USER_AGENT'];
$fastDevices = Array( $fastDevices = Array(
"Mozilla/5.0 (X11;", "Mozilla/5.0 (X11;",
"Mozilla/5.0 (Windows;", "Mozilla/5.0 (Windows;",
"Mozilla/5.0 (iP", "Mozilla/5.0 (iP",
"Mozilla/5.0 (Linux; U; Android", "Mozilla/5.0 (Linux; U; Android",
"Mozilla/4.0 (compatible; MSIE" "Mozilla/4.0 (compatible; MSIE"
); );
$slowDevices = Array( $slowDevices = Array(
"J2ME", "J2ME",
"MIDP", "MIDP",
"Opera/", "Opera/",
"Mozilla/2.0 (compatible;", "Mozilla/2.0 (compatible;",
"Mozilla/3.0 (compatible;" "Mozilla/3.0 (compatible;"
); );
return true; return true;
} }
function array_flatten($a, $f = array()) function array_flatten($a, $f = array())
{ {
if (!$a || !is_array($a)) return ''; if (!$a || !is_array($a)) return '';
foreach ($a as $k => $v) { foreach ($a as $k => $v) {
if (is_array($v)) $f = array_flatten($v, $f); if (is_array($v)) $f = array_flatten($v, $f);
else $f[$k] = $v; else $f[$k] = $v;
} }
return $f; return $f;
} }
function remove_spaces($string) function remove_spaces($string)
{ {
return str_replace(' ', '', $string); return str_replace(' ', '', $string);
} }
function object2array($object) function object2array($object)
{ {
if (is_object($object)) { if (is_object($object)) {
foreach ($object as $key => $value) { foreach ($object as $key => $value) {
$array[$key] = $value; $array[$key] = $value;
} }
} }
else { else {
$array = $object; $array = $object;
} }
return $array; return $array;
} }
function startsWith($haystack, $needle, $case = true) function startsWith($haystack, $needle, $case = true)
{ {
if ($case) { if ($case) {
return (strcmp(substr($haystack, 0, strlen($needle)) , $needle) === 0); return (strcmp(substr($haystack, 0, strlen($needle)) , $needle) === 0);
} }
return (strcasecmp(substr($haystack, 0, strlen($needle)) , $needle) === 0); return (strcasecmp(substr($haystack, 0, strlen($needle)) , $needle) === 0);
} }
   
function endsWith($haystack, $needle, $case = true) function endsWith($haystack, $needle, $case = true)
{ {
if ($case) { if ($case) {
return (strcmp(substr($haystack, strlen($haystack) - strlen($needle)) , $needle) === 0); return (strcmp(substr($haystack, strlen($haystack) - strlen($needle)) , $needle) === 0);
} }
return (strcasecmp(substr($haystack, strlen($haystack) - strlen($needle)) , $needle) === 0); return (strcasecmp(substr($haystack, strlen($haystack) - strlen($needle)) , $needle) === 0);
} }
function bracketsMeanNewLine($input) function bracketsMeanNewLine($input)
{ {
return str_replace(")", "</small>", str_replace("(", "<br><small>", $input)); return str_replace(")", "</small>", str_replace("(", "<br><small>", $input));
} }
function sksort(&$array, $subkey = "id", $sort_ascending = false) function sksort(&$array, $subkey = "id", $sort_ascending = false)
{ {
if (count($array)) $temp_array[key($array) ] = array_shift($array); if (count($array)) $temp_array[key($array) ] = array_shift($array);
foreach ($array as $key => $val) { foreach ($array as $key => $val) {
$offset = 0; $offset = 0;
$found = false; $found = false;
foreach ($temp_array as $tmp_key => $tmp_val) { foreach ($temp_array as $tmp_key => $tmp_val) {
if (!$found and strtolower($val[$subkey]) > strtolower($tmp_val[$subkey])) { if (!$found and strtolower($val[$subkey]) > strtolower($tmp_val[$subkey])) {
$temp_array = array_merge((array)array_slice($temp_array, 0, $offset) , array( $temp_array = array_merge((array)array_slice($temp_array, 0, $offset) , array(
$key => $val $key => $val
) , array_slice($temp_array, $offset)); ) , array_slice($temp_array, $offset));
$found = true; $found = true;
} }
$offset++; $offset++;
} }
if (!$found) $temp_array = array_merge($temp_array, array( if (!$found) $temp_array = array_merge($temp_array, array(
$key => $val $key => $val
)); ));
} }
if ($sort_ascending) $array = array_reverse($temp_array); if ($sort_ascending) $array = array_reverse($temp_array);
else $array = $temp_array; else $array = $temp_array;
} }
function sktimesort(&$array, $subkey = "id", $sort_ascending = false) function sktimesort(&$array, $subkey = "id", $sort_ascending = false)
{ {
if (count($array)) $temp_array[key($array) ] = array_shift($array); if (count($array)) $temp_array[key($array) ] = array_shift($array);
foreach ($array as $key => $val) { foreach ($array as $key => $val) {
$offset = 0; $offset = 0;
$found = false; $found = false;
foreach ($temp_array as $tmp_key => $tmp_val) { foreach ($temp_array as $tmp_key => $tmp_val) {
if (!$found and strtotime($val[$subkey]) > strtotime($tmp_val[$subkey])) { if (!$found and strtotime($val[$subkey]) > strtotime($tmp_val[$subkey])) {
$temp_array = array_merge((array)array_slice($temp_array, 0, $offset) , array( $temp_array = array_merge((array)array_slice($temp_array, 0, $offset) , array(
$key => $val $key => $val
) , array_slice($temp_array, $offset)); ) , array_slice($temp_array, $offset));
$found = true; $found = true;
} }
$offset++; $offset++;
} }
if (!$found) $temp_array = array_merge($temp_array, array( if (!$found) $temp_array = array_merge($temp_array, array(
$key => $val $key => $val
)); ));
} }
if ($sort_ascending && isset($temp_array)) $array = array_reverse($temp_array); if ($sort_ascending && isset($temp_array)) $array = array_reverse($temp_array);
else $array = $temp_array; else $array = $temp_array;
} }
function r_implode( $glue, $pieces ) function r_implode( $glue, $pieces )
{ {
foreach( $pieces as $r_pieces ) foreach( $pieces as $r_pieces )
{ {
if( is_array( $r_pieces ) ) if( is_array( $r_pieces ) )
{ {
$retVal[] = r_implode( $glue, $r_pieces ); $retVal[] = r_implode( $glue, $r_pieces );
} }
else else
{ {
$retVal[] = $r_pieces; $retVal[] = $r_pieces;
} }
} }
return implode( $glue, $retVal ); return implode( $glue, $retVal );
} }
?> ?>
   
<?php <?php
if ( php_sapi_name() == "cli") { if ( php_sapi_name() == "cli") {
include ('include/common.inc.php'); include ('include/common.inc.php');
$conn = pg_connect("dbname=transitdata user=postgres password=snmc host=localhost") or die('connection failed'); $conn = pg_connect("dbname=transitdata user=postgres password=snmc host=localhost") or die('connection failed');
  $pdconn = new PDO("pgsql:dbname=transitdata;user=postgres;password=snmc;host=localhost");
   
   
// Unzip cbrfeed.zip, import all csv files to database // Unzip cbrfeed.zip, import all csv files to database
$unzip = true; $unzip = true;
$zip = zip_open(dirname(__FILE__) . "/cbrfeed.zip"); $zip = zip_open(dirname(__FILE__) . "/cbrfeed.zip");
$tmpdir = "/tmp/cbrfeed/"; $tmpdir = "/tmp/cbrfeed/";
mkdir($tmpdir); mkdir($tmpdir);
if ($unzip) { if ($unzip) {
if (is_resource($zip)) { if (is_resource($zip)) {
while ($zip_entry = zip_read($zip)) { while ($zip_entry = zip_read($zip)) {
$fp = fopen($tmpdir . zip_entry_name($zip_entry) , "w"); $fp = fopen($tmpdir . zip_entry_name($zip_entry) , "w");
if (zip_entry_open($zip, $zip_entry, "r")) { if (zip_entry_open($zip, $zip_entry, "r")) {
echo "Extracting " . zip_entry_name($zip_entry) . "\n"; echo "Extracting " . zip_entry_name($zip_entry) . "\n";
$buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry)); $buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry));
fwrite($fp, "$buf"); fwrite($fp, "$buf");
zip_entry_close($zip_entry); zip_entry_close($zip_entry);
fclose($fp); fclose($fp);
} }
} }
zip_close($zip); zip_close($zip);
} }
} }
foreach (scandir($tmpdir) as $file) { foreach (scandir($tmpdir) as $file) {
if (!strpos($file, ".txt") === false) { if (!strpos($file, ".txt") === false) {
$fieldseparator = ","; $fieldseparator = ",";
$lineseparator = "\n"; $lineseparator = "\n";
$tablename = str_replace(".txt", "", $file); $tablename = str_replace(".txt", "", $file);
echo "Opening $file \n"; echo "Opening $file \n";
$line = 0; $line = 0;
$handle = fopen($tmpdir . $file, "r"); $handle = fopen($tmpdir . $file, "r");
  if ($tablename =="stop_times") {
  $stmt = $pdconn->prepare("insert into stop_times (trip_id,stop_id,stop_sequence) values(:trip_id, :stop_id, :stop_sequence);");
  $stmt->bindParam(':trip_id',$trip_id);
  $stmt->bindParam(':stop_id',$stop_id);
  $stmt->bindParam(':stop_sequence',$stop_sequence);
  }
   
   
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
if ($line > 0) { if ($line == 0) {
   
  } else {
$query = "insert into $tablename values("; $query = "insert into $tablename values(";
$valueCount = 0; $valueCount = 0;
foreach ($data as $value) { foreach ($data as $value) {
$query.=($valueCount >0 ? "','" :"'").pg_escape_string($value); $query.=($valueCount >0 ? "','" :"'").pg_escape_string($value);
$valueCount++; $valueCount++;
} }
if ($tablename == "stops") { if ($tablename == "stops") {
$query.= "', ST_GeographyFromText('SRID=4326;POINT({$data[2]} {$data[0]})'));"; $query.= "', ST_GeographyFromText('SRID=4326;POINT({$data[2]} {$data[0]})'));";
} else { } else {
$query.= "');"; $query.= "');";
} }
if ($tablename =="stop_times" && $data[1] == "") { if ($tablename =="stop_times" && $data[1] == "") {
$query = "insert into $tablename (trip_id,stop_id,stop_sequence) values('{$data[0]}','{$data[3]}','{$data[4]}');"; // $query = "insert into $tablename (trip_id,stop_id,stop_sequence) values('{$data[0]}','{$data[3]}','{$data[4]}');";
} $trip_id=$data[0];
  $stop_id=$data[3];
  $stop_sequence=$data[4];
  }
} }
$result = pg_query($conn, $query); if ($tablename =="stop_times") {
  $stmt->execute();
  }
  else {
  $result = pg_query($conn, $query);
  }
$line++; $line++;
if ($line % 10000 == 0) echo "$line records... \n"; if ($line % 10000 == 0) echo "$line records... ".date('c')."\n";
} }
fclose($handle); fclose($handle);
echo "Found a total of $line records in $file.\n"; echo "Found a total of $line records in $file.\n";
   
} }
} }
} }
?> ?>