Port DB import to PDO to use prepared statements for maximum performance
--- a/include/common-db.inc.php
+++ b/include/common-db.inc.php
@@ -11,9 +11,6 @@
if (!$conn) {
die("A database error occurred.\n");
}
-if (isDebug()) {
- $conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
-}
function databaseError($errMsg)
{
die($errMsg);
--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -29,6 +29,11 @@
$labsPath = "";
if (strstr($_SERVER['PHP_SELF'],"labs")) $labsPath = "../";
+function isDebugServer()
+{
+ return !isset($_SERVER['SERVER_NAME']) || $_SERVER['SERVER_NAME'] == "10.0.1.154" || $_SERVER['SERVER_NAME'] == "10.1.0.4" || $_SERVER['SERVER_NAME'] == "localhost" || $_SERVER['SERVER_NAME'] == "127.0.0.1" ;
+}
+
include_once ("common-geo.inc.php");
include_once ("common-net.inc.php");
include_once ("common-transit.inc.php");
@@ -38,10 +43,7 @@
include_once ("common-session.inc.php");
include_once ("common-template.inc.php");
-function isDebugServer()
-{
- return $_SERVER['SERVER_NAME'] == "10.0.1.154" || $_SERVER['SERVER_NAME'] == "localhost" || $_SERVER['SERVER_NAME'] == "127.0.0.1" || !$_SERVER['SERVER_NAME'];
-}
+
function isAnalyticsOn()
{
return !isDebugServer();
--- a/updatedb.php
+++ b/updatedb.php
@@ -1,59 +1,74 @@
<?php
-include('include/common-db.inc.php');
+include ('include/common.inc.php');
+//$conn = pg_connect("dbname=transitdata user=postgres password=snmc host=localhost") or die('connection failed');
// Unzip cbrfeed.zip, import all csv files to database
-$unzip = true;
+$unzip = false;
$zip = zip_open(dirname(__FILE__) . "/cbrfeed.zip");
$tmpdir = "/tmp/cbrfeed/";
mkdir($tmpdir);
if ($unzip) {
-if (is_resource($zip)) {
- while ($zip_entry = zip_read($zip)) {
- $fp = fopen($tmpdir . zip_entry_name($zip_entry) , "w");
- if (zip_entry_open($zip, $zip_entry, "r")) {
- echo "Extracting " . zip_entry_name($zip_entry) . "\n";
- $buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry));
- fwrite($fp, "$buf");
- zip_entry_close($zip_entry);
- fclose($fp);
+ if (is_resource($zip)) {
+ while ($zip_entry = zip_read($zip)) {
+ $fp = fopen($tmpdir . zip_entry_name($zip_entry) , "w");
+ if (zip_entry_open($zip, $zip_entry, "r")) {
+ echo "Extracting " . zip_entry_name($zip_entry) . "\n";
+ $buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry));
+ fwrite($fp, "$buf");
+ zip_entry_close($zip_entry);
+ fclose($fp);
+ }
}
+ zip_close($zip);
}
- zip_close($zip);
}
-}
-
foreach (scandir($tmpdir) as $file) {
if (!strpos($file, ".txt") === false) {
$fieldseparator = ",";
$lineseparator = "\n";
$tablename = str_replace(".txt", "", $file);
- echo "Opening $file \n";
+ echo "Opening $tmpdir . $file \n";
$line = 0;
$handle = fopen($tmpdir . $file, "r");
- while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
- if ($line > 0) {
+ $stmt = null;
+ $stmt_noarrival = $conn->prepare("insert into stop_times (trip_id,stop_id,stop_sequence) values(? ? ?);");
+ while (($columns = fgetcsv($handle, 1000, ",")) !== FALSE) {
+ if ($line == 0) {
$query = "insert into $tablename values(";
- $valueCount = 0;
- foreach ($data as $value) {
- $query.=($valueCount >0 ? "','" :"'").pg_escape_string($value);
- $valueCount++;
- }
+ $valueCount = 0;
+ foreach ($columns as $value) {
+ $query.= ($valueCount >0 ? "," :"")." ? ";
+ $valueCount++;
+ }
if ($tablename == "stops") {
- $query.= "', ST_GeographyFromText('SRID=4326;POINT({$data[2]} {$data[0]})'));";
- } else {
- $query.= "');";
- }
- if ($tablename =="stop_times" && $data[1] == "") {
- $query = "insert into $tablename (trip_id,stop_id,stop_sequence) values('{$data[0]}','{$data[3]}','{$data[4]}');";
- }
-
+ $query.= ", ST_GeographyFromText('SRID=4326;POINT(? ?)'));";
+ }
+ else {
+ $query.= ");";
+ }
+ $stmt = $conn->prepare($query);
}
- $result = pg_query($conn, $query);
+ else {
+ $data = $columns;
+ if ($tablename == "stops") {
+ $data[] = $data[2];
+ $data[] = $data[0];
+ }
+ if ($tablename == "stop_times" && $data[1] == "") {
+ $stmt_noarrival->execute(Array(
+ $data[0],
+ $data[3],
+ $data[4]
+ ));
+ }
+ else {
+ $stmt->execute($data);
+ }
+ }
$line++;
- if ($line % 10000 == 0) echo "$line records... \n";
+ if ($line % 10000 == 0) echo "$line records... \n";
}
fclose($handle);
echo "Found a total of $line records in $file.\n";
-
}
}
?>