From: maxious Date: Fri, 15 Apr 2011 14:08:30 +0000 Subject: Service warnings for changed timetable days X-Git-Url: http://maxious.lambdacomplex.org/git/?p=busui.git&a=commitdiff&h=a0bda5d898199a4d51fd890589fa3d284a72ba43 --- Service warnings for changed timetable days --- --- /dev/null +++ b/aws/awsStartup.sh @@ -1,1 +1,37 @@ +#!/bin/bash +#this script should be run from a fresh git checkout from github +#ami base must have yum install lighttpd-fastcgi, git, tomcat6 +#php-cli php-gd tomcat6-webapps tomcat6-admin-webapps svn maven2 +#postgres postgres-server php-pg +#http://www.how2forge.org/installing-lighttpd-with-php5-and-mysql-support-on-fedora-12 +cp /root/aws.php /tmp/ +mkdir /var/www/lib/staticmaplite/cache +chcon -h system_u:object_r:httpd_sys_content_t /var/www +chcon -R -h root:object_r:httpd_sys_content_t /var/www/* +chcon -R -t httpd_sys_content_rw_t /var/www/lib/staticmaplite/cache +chmod -R 777 /var/www/lib/staticmaplite/cache +wget http://s3-ap-southeast-1.amazonaws.com/busresources/cbrfeed.zip \ +-O /var/www/cbrfeed.zip + +createdb transitdata +createlang -d transitdata plpgsql +psql -d transitdata -f /var/www/lib/postgis.sql +# curl https://github.com/maxious/ACTBus-ui/raw/master/transitdata.cbrfeed.sql.gz -o transitdata.cbrfeed.sql.gz +#made with pg_dump transitdata | gzip -c > transitdata.cbrfeed.sql.gz +gunzip /var/www/transitdata.cbrfeed.sql.gz +psql -d transitdata -f /var/www/transitdata.cbrfeed.sql +#createuser transitdata -SDRP +#password transitdata +#psql -d transitdata -c \"GRANT SELECT ON TABLE agency,calendar,calendar_dates,routes,stop_times,stops,trips TO transitdata;\" +php /var/www/updatedb.php + +wget http://s3-ap-southeast-1.amazonaws.com/busresources/Graph.obj \ +-O /tmp/Graph.obj +rm -rfv /usr/share/tomcat6/webapps/opentripplanner* +wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-webapp.war \ +-O /usr/share/tomcat6/webapps/opentripplanner-webapp.war +wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-api-webapp.war \ +-O /usr/share/tomcat6/webapps/opentripplanner-api-webapp.war +/etc/init.d/tomcat6 restart + --- /dev/null +++ b/aws/pg_hba.conf @@ -1,1 +1,75 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the +# PostgreSQL documentation for a complete description +# of this file. A short synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain socket, +# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an +# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", a database name, or +# a comma-separated list thereof. +# +# USER can be "all", a user name, a group name prefixed with "+", or +# a comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names from +# a separate file. +# +# CIDR-ADDRESS specifies the set of hosts the record matches. +# It is made up of an IP address and a CIDR mask that is an integer +# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies +# the number of significant bits in the mask. Alternatively, you can write +# an IP address and netmask in separate columns to specify the set of hosts. +# +# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", "krb5", +# "ident", "pam", "ldap" or "cert". Note that "password" sends passwords +# in clear text; "md5" is preferred since it sends encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different authentication +# methods - refer to the "Client Authentication" section in the documentation +# for a list of which options are available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other special +# characters must be quoted. Quoting one of the keywords "all", "sameuser" or +# "samerole" makes the name lose its special character, and just match a +# database or username with that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL listen +# on a non-local interface via the listen_addresses configuration parameter, +# or via the -i or -h command line switches. +# + + + +# TYPE DATABASE USER CIDR-ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust + --- a/feedback.php +++ b/feedback.php @@ -48,7 +48,7 @@ if you click on feedback from a stop page, these will get filled in automatically. else describe the location/street of the stop in one of these boxes
Suggested Stop Location (lat/long or words):
- if your device supports javascript, you can pick a location from the map above
+ --- /dev/null +++ b/include/common-db.inc.php @@ -1,1 +1,21 @@ + --- a/include/common-session.inc.php +++ b/include/common-session.inc.php @@ -53,4 +53,7 @@ } debug(print_r($_SESSION, true) , "session"); +function current_time() { + return ($_SESSION['time']? $_SESSION['time'] : date("H:i:s")); +} ?> --- a/include/common-template.inc.php +++ b/include/common-template.inc.php @@ -74,10 +74,14 @@ .ui-listview-filter { margin: 0 !important; } - .ui-icon-navigation { + .ui-icon-navigation { background-image: url(css/images/113-navigation.png); background-position: 1px 0; } + .ui-icon-beaker { + background-image: url(css/images/91-beaker-2.png); + background-position: 1px 0; + } #footer { text-size: 0.75em; text-align: center; @@ -98,6 +102,14 @@ #extrainfo { visibility: hidden; display: none; + } + #servicewarning { + padding: 1em; + margin-bottom: 0.5em; + text-size: 0.2em; + background-color: #FF9; + -moz-border-radius: 15px; +border-radius: 15px; } // source http://webaim.org/techniques/skipnav/ #skip a, #skip a:hover, #skip a:visited @@ -169,12 +181,6 @@ '; if ($opendiv) { echo '
-
Back

' . $pageTitle . '

@@ -182,6 +188,12 @@
'; + if (!$_SESSION['service_id']) { + $overrides = getServiceOverride(); + if ($overrides['service_id']){ + echo '
Buses are running on an altered timetable today due to industrial action/public holiday. See http://www.action.act.gov.au for details.
'; + } + } } } function include_footer() @@ -224,7 +236,7 @@
- Current Time? + Current Time?
--- a/include/common-transit.inc.php +++ b/include/common-transit.inc.php @@ -4,9 +4,26 @@ 'saturday', 'weekday' ); +function getServiceOverride() { + global $conn; + $query = "Select * from calendar_dates where date = '".date("Ymd")."' and exception_type = '1'"; + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_assoc($result); +} function service_period() { + if (isset($_SESSION['service_period'])) return $_SESSION['service_period']; + $override = getServiceOverride(); + if ($override['service_id']){ + return $override['service_id']; + } + switch (date('w')) { case 0: return 'sunday'; --- a/include/db/route-dao.inc.php +++ b/include/db/route-dao.inc.php @@ -40,23 +40,49 @@ } function getRouteNextTrip($routeID) { - global $conn; + global $conn; $query = "select * from routes join trips on trips.route_id = routes.route_id join stop_times on stop_times.trip_id = trips.trip_id where -arrival_time > CURRENT_TIME and routes.route_id = '$routeID' order by +arrival_time > '".current_time()."' and routes.route_id = '$routeID' order by arrival_time limit 1"; debug($query,"database"); $result = pg_query($conn, $query); - if (!$result) { + if (!$result) { databaseError(pg_result_error($result)); return Array(); } - return pg_fetch_assoc($result); + $r = pg_fetch_assoc($result); + // past last trip of the day special case + if (sizeof($r) == 0) { + $query = "select * from routes join trips on trips.route_id = routes.route_id +join stop_times on stop_times.trip_id = trips.trip_id where routes.route_id = '$routeID' order by +arrival_time DESC limit 1"; + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + $r = pg_fetch_assoc($result); + } + return $r; } + + function getTimeInterpolatedRouteAtStop($routeID, $stop_id) +{ + $nextTrip = getRouteNextTrip($routeID); + if ($nextTrip['trip_id']){ + foreach (getTimeInterpolatedTrip($nextTrip['trip_id']) as $tripStop) { + if ($tripStop['stop_id'] == $stop_id) return $tripStop; + } + } + return Array(); +} + function getRouteTrips($routeID) { global $conn; - $query = "select * from routes join trips on trips.route_id = routes.route_id -join stop_times on stop_times.trip_id = trips.trip_id where routes.route_id = '$routeID' order by + $query = "select routes.route_id,trips.trip_id,service_id,arrival_time, stop_id, stop_sequence from routes join trips on trips.route_id = routes.route_id +join stop_times on stop_times.trip_id = trips.trip_id where routes.route_id = '$routeID' and stop_sequence = '1' order by arrival_time "; debug($query,"database"); $result = pg_query($conn, $query); @@ -112,7 +138,7 @@ if ($service_period == "") $service_period = service_period(); if ($limit != "") $limit = " LIMIT $limit "; global $conn; - $query = "SELECT service_id,trips.route_id,route_short_name,route_long_name, + $query = "SELECT service_id,trips.route_id,route_short_name,route_long_name,min(stops.stop_id) as stop_id, min(ST_Distance(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), FALSE)) as distance FROM stop_times join trips on trips.trip_id = stop_times.trip_id --- a/include/db/stop-dao.inc.php +++ b/include/db/stop-dao.inc.php @@ -84,18 +84,17 @@ $afterCondition = "AND arrival_time > '$afterTime'"; global $conn; if ($afterTime != "") { - $query = " SELECT stop_times.trip_id,stop_times.arrival_time,stop_times.stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name, start_times.arrival_time as start_time + $query = " SELECT stop_times.trip_id,stop_times.arrival_time,stop_times.stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name, end_times.arrival_time as end_time FROM stop_times join trips on trips.trip_id = stop_times.trip_id -join routes on trips.route_id = routes.route_id , (SELECT trip_id,arrival_time from stop_times - WHERE stop_times.arrival_time IS NOT NULL - AND stop_sequence = '1') as start_times +join routes on trips.route_id = routes.route_id , (SELECT trip_id,max(arrival_time) as arrival_time from stop_times + WHERE stop_times.arrival_time IS NOT NULL group by trip_id) as end_times WHERE stop_times.stop_id = '$stopID' -AND stop_times.trip_id = start_times.trip_id +AND stop_times.trip_id = end_times.trip_id AND service_id='$service_period' -AND start_times.arrival_time > '$afterTime' -ORDER BY start_time"; +AND end_times.arrival_time > '$afterTime' +ORDER BY end_time"; } else { $query = "SELECT stop_times.trip_id,arrival_time,stop_times.stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name @@ -115,15 +114,16 @@ } return pg_fetch_all($result); } -function getStopTripsWithTimes($stopID, $time = "", $service_period = "", $time_range = "") +function getStopTripsWithTimes($stopID, $time = "", $service_period = "", $time_range = "", $limit = "") { if ($service_period == "") $service_period = service_period(); if ($time_range == "") $time_range = (24 * 60 * 60); - if ($time == "") $time = ($_SESSION['time'] ? $_SESSION['time'] : date("H:i:s")); + if ($time == "") $time = current_time(); if ($limit == "") $limit = 10; $trips = getStopTrips($stopID, $service_period, $time); $timedTrips = Array(); - foreach ($trips as $trip) { + if ($trips && sizeof($trips) > 0) { + foreach ($trips as $trip) { if ($trip['arrival_time'] != "") { if (strtotime($trip['arrival_time']) > strtotime($time) and strtotime($trip['arrival_time']) < (strtotime($time) + $time_range)) { $timedTrips[] = $trip; @@ -138,6 +138,7 @@ if (sizeof($timedTrips) > $limit) break; } sktimesort($timedTrips, "arrival_time", true); + } return $timedTrips; } ?> --- a/include/db/trip-dao.inc.php +++ b/include/db/trip-dao.inc.php @@ -49,7 +49,7 @@ points.append((stop.stop_lat, stop.stop_lon)) return points*/ } -function getTimeInterpolatedTrip($tripID) +function getTimeInterpolatedTrip($tripID, $range = "") { global $conn; $query = "SELECT stop_times.trip_id,arrival_time,stop_times.stop_id,stop_lat,stop_lon,stop_name,stop_code, @@ -58,7 +58,7 @@ join trips on trips.trip_id = stop_times.trip_id join routes on trips.route_id = routes.route_id join stops on stops.stop_id = stop_times.stop_id -WHERE trips.trip_id = '$tripID' ORDER BY stop_sequence"; +WHERE trips.trip_id = '$tripID' $range ORDER BY stop_sequence"; debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { @@ -111,7 +111,18 @@ } function getTimeInterpolatedTripAtStop($tripID, $stop_sequence) { - foreach (getTimeInterpolatedTrip($tripID) as $tripStop) { + global $conn; + // limit interpolation to between nearest actual points. + $prevTimePoint = pg_fetch_assoc(pg_query($conn," SELECT trip_id,stop_id, + stop_sequence +FROM stop_times +WHERE trip_id = '$tripID' and stop_sequence < $stop_sequence and stop_times.arrival_time IS NOT NULL ORDER BY stop_sequence DESC LIMIT 1")); + $nextTimePoint = pg_fetch_assoc(pg_query($conn," SELECT trip_id,stop_id, + stop_sequence +FROM stop_times +WHERE trip_id = '$tripID' and stop_sequence > $stop_sequence and stop_times.arrival_time IS NOT NULL ORDER BY stop_sequence LIMIT 1")); + $range = "AND stop_sequence >= '{$prevTimePoint['stop_sequence']}' AND stop_sequence <= '{$nextTimePoint['stop_sequence']}'"; + foreach (getTimeInterpolatedTrip($tripID,$range) as $tripStop) { if ($tripStop['stop_sequence'] == $stop_sequence) return $tripStop; } return Array(); @@ -132,6 +143,22 @@ $r = pg_fetch_assoc($result); return $r['arrival_time']; } +function getActiveTrips($time) +{ + global $conn; + if ($time == "") $time = current_time(); + $query = "Select distinct stop_times.trip_id, start_times.arrival_time as start_time, end_times.arrival_time as end_time from stop_times, (SELECT trip_id,arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL +AND stop_sequence = '1') as start_times, (SELECT trip_id,max(arrival_time) as arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL group by trip_id) as end_times +WHERE start_times.trip_id = end_times.trip_id AND stop_times.trip_id = end_times.trip_id AND $time > start_times.arrival_time AND $time < end_times.arrival_time"; + debug($query, "database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); +} + function viaPointNames($tripid, $stop_sequence = "") { global $conn; --- a/index.php +++ b/index.php @@ -25,6 +25,7 @@ Busness R&D'; include_footer(true) ?> --- /dev/null +++ b/labs/css @@ -1,1 +1,1 @@ - +../css/ --- /dev/null +++ b/labs/index.php @@ -1,1 +1,16 @@ + + +
+ + --- /dev/null +++ b/labs/js @@ -1,1 +1,1 @@ - +../js --- /dev/null +++ b/labs/lib @@ -1,1 +1,1 @@ - +../lib --- a/labs/myway_api.json.php +++ b/labs/myway_api.json.php @@ -74,7 +74,7 @@ } if (!isset($return['error'])) { - include_once ('simple_html_dom.php'); + include_once ('lib/simple_html_dom.php'); $page = str_get_html($pageHTML); $pageAlerts = $page->find(".smartCardAlert"); if (sizeof($pageAlerts) > 0) { --- a/labs/mywaybalance.php +++ b/labs/mywaybalance.php @@ -1,21 +1,36 @@ +
+ Back +

MyWay Balance

+ Logout +
+ +
'; + $return = Array(); +function logout() { + setcookie("card_number", "", time() - 60 * 60 * 24 * 100, "/"); + setcookie("date", "", time() - 60 * 60 * 24 * 100, "/"); + setcookie("secret_answer", "", time() - 60 * 60 * 24 * 100, "/"); +} function printBalance($cardNumber, $date, $pwrd) { global $return; - $return = json_decode(getPage(curPageURL() . "/myway_api.json.php?card_number=$cardNumber&DOBday={$date[0]}&DOBmonth={$date[1]}&DOByear={$date[2]}&secret_answer=$pwrd"), true); - - if (isset($return['error'])) { - echo "" . var_dump($return['error']) . ""; - } else { + $return = json_decode(getPage(curPageURL() . "/myway_api.json.php?card_number=$cardNumber&DOBday={$date[0]}&DOBmonth={$date[1]}&DOByear={$date[2]}&secret_answer=$pwrd") , true); + if (isset($return['error'])) { + logout(); + echo '

' . $return['error'][0] . "

"; + } + else { echo "

Balance: " . $return['myway_carddetails']['Card Balance'] . "

"; echo '
  • Recent Transactions
  • '; foreach ($return['myway_transactions'] as $transaction) { echo "
  • " . $transaction["Date / Time"] . ""; - echo "
    " . $transaction["TX Reference No / Type"]. ""; - echo '

    '.$transaction["TX Amount"].'

    '; + echo "
    " . $transaction["TX Reference No / Type"] . ""; + echo '

    ' . $transaction["TX Amount"] . '

    '; echo "
  • "; } echo "
"; @@ -25,12 +40,15 @@ $cardNumber = $_REQUEST['card_number']; $date = explode("/", $_REQUEST['date']); $pwrd = $_REQUEST['secret_answer']; - if ($_REQUEST['remember'] == true) { - $_COOKIE['card_number'] = $cardNumber; - $_COOKIE['date'] = $date; - $_COOKIE['secret_answer'] = $pwrd; + if ($_REQUEST['remember'] == "on") { + setcookie("card_number", $cardNumber, time() + 60 * 60 * 24 * 100, "/"); + setcookie("date", $_REQUEST['date'], time() + 60 * 60 * 24 * 100, "/"); + setcookie("secret_answer", $pwrd, time() + 60 * 60 * 24 * 100, "/"); } printBalance($cardNumber, $date, $pwrd); +} +else if (isset($_REQUEST['logout'])) { + echo '

Logged out of MyWay balance

Back to main menu...
'; } else if (isset($_COOKIE['card_number']) && isset($_COOKIE['date']) && isset($_COOKIE['secret_answer'])) { $cardNumber = $_COOKIE['card_number']; --- a/labs/tripPlannerTester.kml.php +++ b/labs/tripPlannerTester.kml.php @@ -8,6 +8,50 @@ } else { return (($pBegin - $pEnd) * (1 - ($pStep / $pMax))) + $pEnd; + } +} +require ("../lib/rolling-curl/RollingCurl.php"); +function processResult_cb($response, $info, $request) +{ + global $testRegions, $regionTimes,$csv,$kml, $latdeltasize,$londeltasize; + $md = $request->metadata; + $tripplan = json_decode($response); + $plans = Array(); + //var_dump(Array($info, $request)); + if (is_array($tripplan->plan->itineraries->itinerary)) { + foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) { + $plans[floor($itinerary->duration / 60000) ] = $itinerary; + } + } + else { + $plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary; + } + if ($csv) echo "{$md['i']},{$md['j']}," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,{$md['key']}\n"; + if ($kml) { + $time = min(array_keys($plans)); + $plan = ""; + if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) { + foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) { + $plan.= processLeg($legNumber, $leg) . ","; + } + } + else { + $plan.= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg); + } + if (isset($tripplan->error) && $tripplan->error->id == 404) { + $time = 999; + $plan = "Trip not possible without excessive walking from nearest bus stop"; + } + $testRegions[] = Array( + "lat" => $md['i'], + "lon" => $md['j'], + "time" => $time, + "latdeltasize" => $latdeltasize, + "londeltasize" => $londeltasize, + "regionname" => $md['key'], + "plan" => $plan . "
original plan" + ); + $regionTimes[] = $time; } } function Gradient($HexFrom, $HexTo, $ColorSteps) @@ -49,18 +93,17 @@ //} //$walkingstep.= floor($step->distance) . "m"; //return $walkingstep; + } } $csv = false; $kml = true; if ($kml) { - //header('Content-Type: application/vnd.google-earth.kml+xml'); + header('Content-Type: application/vnd.google-earth.kml+xml'); echo ' '; } include ('../include/common.inc.php'); -//Test code to grab transit times -// make sure to sleep(10); $boundingBoxes = Array( "belconnen" => Array( "startlat" => - 35.1928, @@ -105,72 +148,29 @@ $useragent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1"; if ($csv) echo "
";
 if ($csv) echo "lat,lon,time,latdeltasize, londeltasize, region key name\n";
+$rc = new RollingCurl("processResult_cb");
+$rc->window_size = 3;
 foreach ($boundingBoxes as $key => $boundingBox) {
 	for ($i = $boundingBox['startlat']; $i >= $boundingBox['finishlat']; $i-= $latdeltasize) {
 		for ($j = $boundingBox['startlon']; $j <= $boundingBox['finishlon']; $j+= $londeltasize) {
 			$url = $otpAPIurl . "ws/plan?date=" . urlencode($startDate) . "&time=" . urlencode($startTime) . "&mode=TRANSIT%2CWALK&optimize=QUICK&maxWalkDistance=440&wheelchair=false&toPlace=" . $i . "," . $j . "&fromPlace=$fromPlace";
-			$ch = curl_init($url);
-			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
-			curl_setopt($ch, CURLOPT_HEADER, 0);
-			curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+			$request = new RollingCurlRequest($url);
+			$request->headers = Array(
 				"Accept: application/json"
-			));
-			curl_setopt($ch, CURLOPT_TIMEOUT, 5);
-			$page = curl_exec($ch);
-			if (curl_errno($ch)) {
-				if ($csv) echo "Trip planner temporarily unavailable: " . curl_errno($ch) . " " . curl_error($ch);
-			}
-			else {
-				$tripplan = json_decode($page); 
-				$plans = Array();
-				if (is_array($tripplan->plan->itineraries->itinerary)) {
-					foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) {
-						$plans[floor($itinerary->duration / 60000) ] = $itinerary;
-					}
-				}
-				else {
-					$plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary;
-				}
-				if ($csv) echo "$i,$j," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,$key\n";
-				if ($kml) {
-					$time = min(array_keys($plans));
-					$plan = "";
-					if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) {
-						foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) {
-							$plan .= processLeg($legNumber, $leg).",";
-						}
-					}
-					else {
-						$plan .= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg);
-					}
-						if (isset($tripplan->error) && $tripplan->error->id == 404) {
-							$time = 999;
-							$plan = "Trip not possible without excessive walking from nearest bus stop";
-						}
-					$testRegions[] = Array(
-						"lat" => $i,
-						"lon" => $j,
-						"time" => $time,
-						"latdeltasize" => $latdeltasize,
-						"londeltasize" => $londeltasize,
-						"regionname" => $key,
-						"plan" => $plan . "
original plan" - ); - $regionTimes[] = $time; - } - } - flush(); @ob_flush(); - curl_close($ch); - } - } -} + ); + $request->metadata = Array( "i" => $i, "j" => $j, "key" => $key); + $rc->add($request); + } + } +} +$rc->execute(); if ($kml) { $colorSteps = 9; //$minTime = min($regionTimes); //$maxTime = max($regionTimes); //$rangeTime = $maxTime - $minTime; //$deltaTime = $rangeTime / $colorSteps; - $Gradients = Gradient(strrev("66FF00"), strrev("FF0000"), $colorSteps); // KML is BGR not RGB so strrev + $Gradients = Gradient(strrev("66FF00") , strrev("FF0000") , $colorSteps); // KML is BGR not RGB so strrev foreach ($testRegions as $testRegion) { //$band = (floor(($testRegion[time] - $minTime) / $deltaTime)); $band = (floor($testRegion[time] / 10)); --- a/layar_api.php +++ b/layar_api.php @@ -9,32 +9,30 @@ $page_end = $max_page + filter_var($_REQUEST['pageKey'], FILTER_SANITIZE_NUMBER_INT); $lat = filter_var($_REQUEST['lat'], FILTER_SANITIZE_NUMBER_FLOAT, FILTER_FLAG_ALLOW_FRACTION); $lon = filter_var($_REQUEST['lon'], FILTER_SANITIZE_NUMBER_FLOAT, FILTER_FLAG_ALLOW_FRACTION); - -$contents = getStopsNearby($lat, $lon, 50); - +$max_distance = filter_var($_REQUEST['radius'], FILTER_SANITIZE_NUMBER_INT); +$contents = getNearbyStops($lat, $lon, 50, $max_distance); $stopNum = 0; foreach ($contents as $stop) { $stopNum++; if ($stopNum > $page_start && $stopNum <= $page_end) { $hotspot = Array(); - $hotspot['id'] = $stop[id]; - $hotspot['title'] = $stop[name]; + $hotspot['id'] = $stop['stop_id']; + $hotspot['title'] = $stop['stop_name']; $hotspot['type'] = 0; - $hotspot['lat'] = floor($stop[lat] * 1000000); - $hotspot['lon'] = floor($stop[lon] * 1000000); - $hotspot['distance'] = distance($stop[lat], $stop[lon], $_REQUEST['lat'], $_REQUEST['lon']); + $hotspot['lat'] = floor($stop['stop_lat'] * 1000000); + $hotspot['lon'] = floor($stop['stop_lon'] * 1000000); + $hotspot['distance'] = floor($stop['distance']); + $hotspot['attribution'] = "ACTION Buses"; $hotspot['actions'] = Array( Array( "label" => 'View more trips/information', - 'uri' => 'http://bus.lambdacomplex.org/' . 'stop.php?stopid=' . $stop[id] + 'uri' => 'http://bus.lambdacomplex.org/' . 'stop.php?stopid=' . $stop['stop_id'] ) ); - - $url = $APIurl . "/json/stoptrips?stop=" . $row[0] . "&time=" . midnight_seconds() . "&service_period=" . service_period() . "&limit=4&time_range=" . strval(90 * 60); - $trips = getStopTrips($stopID); + $trips = getStopTripsWithTimes($stop['stop_id'], "", "", "", 3); foreach ($trips as $key => $row) { if ($key < 3) { - $hotspot['line' . strval($key + 2) ] = $row[1][1] . ' @ ' . midnight_seconds_to_time($row[0]); + $hotspot['line' . strval($key + 2) ] = $row['route_short_name'] . ' ' . $row['route_long_name'] . ' @ ' . $row['arrival_time']; } } if (sizeof($trips) == 0) $hotspot['line2'] = 'No trips in the near future.'; @@ -49,7 +47,7 @@ $output['errorString'] = 'no results, try increasing range'; $output['errorCode'] = 21; } -if ($page_end >= $max_results || sizeof($hotspot) < $max_page) { +if ($page_end >= $max_results || sizeof($contents) < $page_start+$max_page) { $output["morePages"] = false; $output["nextPageKey"] = null; } --- /dev/null +++ b/lib/rolling-curl/.svn/all-wcprops @@ -1,1 +1,42 @@ +K 25 +svn:wc:ra_dav:version-url +V 22 +/svn/!svn/ver/20/trunk +END +RollingCurlGroup.php +K 25 +svn:wc:ra_dav:version-url +V 43 +/svn/!svn/ver/20/trunk/RollingCurlGroup.php +END +example_groups.php +K 25 +svn:wc:ra_dav:version-url +V 41 +/svn/!svn/ver/20/trunk/example_groups.php +END +example.php +K 25 +svn:wc:ra_dav:version-url +V 34 +/svn/!svn/ver/20/trunk/example.php +END +RollingCurl.php +K 25 +svn:wc:ra_dav:version-url +V 38 +/svn/!svn/ver/20/trunk/RollingCurl.php +END +CHANGELOG.txt +K 25 +svn:wc:ra_dav:version-url +V 36 +/svn/!svn/ver/20/trunk/CHANGELOG.txt +END +README.txt +K 25 +svn:wc:ra_dav:version-url +V 33 +/svn/!svn/ver/20/trunk/README.txt +END --- /dev/null +++ b/lib/rolling-curl/.svn/entries @@ -1,1 +1,233 @@ - +10 + +dir +20 +http://rolling-curl.googlecode.com/svn/trunk +http://rolling-curl.googlecode.com/svn + + + +2010-09-12T20:39:22.711474Z +20 +alexander.makarow + + + + + + + + + + + + + + +74aa2acc-2e27-11de-b2a4-4f96ceaaac44 + +RollingCurlGroup.php +file + + + + +2011-04-10T08:32:48.081650Z +73c08d9e9e24b4adc89816624c7aca30 +2010-09-12T20:39:22.711474Z +20 +alexander.makarow +has-props + + + + + + + + + + + + + + + + + + + + +5152 + +example_groups.php +file + + + + +2011-04-10T08:32:48.082650Z +907ed82a47d346c39acbd5578e1d0230 +2010-09-12T20:39:22.711474Z +20 +alexander.makarow +has-props + + + + + + + + + + + + + + + + + + + + +1367 + +example.php +file + + + + +2011-04-10T08:32:48.083650Z +87aa845abfaffc09ed4eca024f2a8b8a +2010-09-12T20:39:22.711474Z +20 +alexander.makarow + + + + + + + + + + + + + + + + + + + + + +1860 + +RollingCurl.php +file + + + + +2011-04-10T08:32:48.084650Z +205391c449f3f3ee050004dadc374dc8 +2010-09-12T20:39:22.711474Z +20 +alexander.makarow + + + + + + + + + + + + + + + + + + + + + +10444 + +CHANGELOG.txt +file + + + + +2011-04-10T08:32:48.085650Z +d0452f6f9530ed04580159121d0fd5f7 +2010-09-12T20:39:22.711474Z +20 +alexander.makarow +has-props + + + + + + + + + + + + + + + + + + + + +662 + +README.txt +file + + + + +2011-04-10T08:32:48.085650Z +60dd357081431c0f2b82989cdbce8615 +2010-09-12T20:39:22.711474Z +20 +alexander.makarow + + + + + + + + + + + + + + + + + + + + + +6355 + + --- /dev/null +++ b/lib/rolling-curl/.svn/prop-base/CHANGELOG.txt.svn-base @@ -1,1 +1,6 @@ +K 13 +svn:eol-style +V 6 +native +END --- /dev/null +++ b/lib/rolling-curl/.svn/prop-base/RollingCurlGroup.php.svn-base @@ -1,1 +1,6 @@ +K 13 +svn:eol-style +V 6 +native +END --- /dev/null +++ b/lib/rolling-curl/.svn/prop-base/example_groups.php.svn-base @@ -1,1 +1,6 @@ +K 13 +svn:eol-style +V 6 +native +END --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/CHANGELOG.txt.svn-base @@ -1,1 +1,15 @@ +Rolling Curl changelog +====================== +September 13, 2010 +------------------ +- Bug #12, #14: Fixed default options overriding (LionsAd) +- Bug #10: Added use of curl_multi_select to avoid burning CPU (LionsAd) +- Enh #6, #9: Added $request as parameter to callback function (LionsAd) +- Chg: Request renamed to RollingCurlRequest (LionsAd) +- Added RollingCurlGroup class that allows processing groups of requests (LionsAd) +- More cleanup at unsetting a class (LionsAd) +- Timeout parameter for curl_multi_select is now configurable (LionsAd) +- single_curl now returns true (LionsAd) +- Readme corrections (Alexander Makarov) +- Code cleanup (Alexander Makarov) --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/README.txt.svn-base @@ -1,1 +1,210 @@ - +Rolling Curl +============ + +RollingCurl allows you to process multiple HTTP requests in parallel using CURL PHP library. + +Released under the Apache License 2.0. + +Authors +------- +- Was originally written by [Josh Fraser](joshfraser.com). +- Currently maintained by [Alexander Makarov](http://rmcreative.ru/). +- Received significant updates and patched from [LionsAd](http://github.com/LionsAd/rolling-curl). + +Overview +-------- +RollingCurl is a more efficient implementation of curl_multi() curl_multi is a great way to process multiple HTTP requests in parallel in PHP. +curl_multi is particularly handy when working with large data sets (like fetching thousands of RSS feeds at one time). Unfortunately there is +very little documentation on the best way to implement curl_multi. As a result, most of the examples around the web are either inefficient or +fail entirely when asked to handle more than a few hundred requests. + +The problem is that most implementations of curl_multi wait for each set of requests to complete before processing them. If there are too many requests +to process at once, they usually get broken into groups that are then processed one at a time. The problem with this is that each group has to wait for +the slowest request to download. In a group of 100 requests, all it takes is one slow one to delay the processing of 99 others. The larger the number of +requests you are dealing with, the more noticeable this latency becomes. + +The solution is to process each request as soon as it completes. This eliminates the wasted CPU cycles from busy waiting. Also there is a queue of +cURL requests to allow for maximum throughput. Each time a request is completed, a new one is added from the queue. By dynamically adding and removing +links, we keep a constant number of links downloading at all times. This gives us a way to throttle the amount of simultaneous requests we are sending. +The result is a faster and more efficient way of processing large quantities of cURL requests in parallel. + +Callbacks +--------- + +Each of requests usually do have a callback to process results that is being executed when request is done +(both successfully or not). + +Callback accepts three parameters and can look like the following one: +~~~ +[php] +function request_callback($response, $info, $request){ + // doing something with the data received +} +~~~ + +- $response contains received page body. +- $info is an associative array that holds various information about response such as HTTP response code, content type, +time taken to make request etc. +- $request contains RollingCurlRequest that was used to make request. + +Examples +-------- +### Hello world + +~~~ +[php] +// an array of URL's to fetch +$urls = array("http://www.google.com", + "http://www.facebook.com", + "http://www.yahoo.com"); + +// a function that will process the returned responses +function request_callback($response, $info, $request) { + // parse the page title out of the returned HTML + if (preg_match("~(.*?)~i", $response, $out)) { + $title = $out[1]; + } + echo "$title
"; + print_r($info); + echo "
"; +} + +// create a new RollingCurl object and pass it the name of your custom callback function +$rc = new RollingCurl("request_callback"); +// the window size determines how many simultaneous requests to allow. +$rc->window_size = 20; +foreach ($urls as $url) { + // add each request to the RollingCurl object + $request = new RollingCurlRequest($url); + $rc->add($request); +} +$rc->execute(); +~~~ + + +### Setting custom options + +Set custom options for EVERY request: + +~~~ +[php] +$rc = new RollingCurl("request_callback"); +$rc->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true); +$rc->execute(); +~~~ + +Set custom options for A SINGLE request: + +~~~ +[php] +$rc = new RollingCurl("request_callback"); +$request = new RollingCurlRequest($url); +$request->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true); +$rc->add($request); +$rc->execute(); +~~~ + +### Shortcuts + +~~~ +[php] +$rc = new RollingCurl("request_callback"); +$rc->get("http://www.google.com"); +$rc->get("http://www.yahoo.com"); +$rc->execute(); +~~~ + +### Class callbacks + +~~~ +[php] +class MyInfoCollector { + private $rc; + + function __construct(){ + $this->rc = new RollingCurl(array($this, 'processPage')); + } + + function processPage($response, $info, $request){ + //... + } + + function run($urls){ + foreach ($urls as $url){ + $request = new RollingCurlRequest($url); + $this->rc->add($request); + } + $this->rc->execute(); + } +} + +$collector = new MyInfoCollector(); +$collector->run(array( + 'http://google.com/', + 'http://yahoo.com/' +)); +~~~ + +### Using RollingCurlGroup + +~~~ +[php] +class TestCurlRequest extends RollingCurlGroupRequest { + public $test_verbose = true; + + function process($output, $info) { + echo "Processing " . $this->url . "\n"; + if ($this->test_verbose) + print_r($info); + + parent::process($output, $info); + } +} + +class TestCurlGroup extends RollingCurlGroup { + function process($output, $info, $request) { + echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n"; + parent::process($output, $info, $request); + } + + function finished() { + echo "Group CB: Finished" . $this->name . "\n"; + parent::finished(); + } +} + +$group = new TestCurlGroup("High"); +$group->add(new TestCurlRequest("www.google.de")); +$group->add(new TestCurlRequest("www.yahoo.de")); +$group->add(new TestCurlRequest("www.newyorktimes.com")); +$reqs[] = $group; + +$group = new TestCurlGroup("Normal"); +$group->add(new TestCurlRequest("twitter.com")); +$group->add(new TestCurlRequest("www.bing.com")); +$group->add(new TestCurlRequest("m.facebook.com")); +$reqs[] = $group; + +$reqs[] = new TestCurlRequest("www.kernel.org"); + +// No callback here, as its done in Request class +$rc = new GroupRollingCurl(); + +foreach ($reqs as $req) +$rc->add($req); + +$rc->execute(); +~~~ + +The same function (add) can be used both for adding requests and groups of requests. +The "callback" in request and groups is: + +process($output, $info) + +and + +process($output, $info, $request) + +Also you can override RollingCurlGroup::finished() that will be executed right after finishing group processing. + +$Id$ --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/RollingCurl.php.svn-base @@ -1,1 +1,375 @@ - +url = $url; + $this->method = $method; + $this->post_data = $post_data; + $this->headers = $headers; + $this->options = $options; + } + + /** + * @return void + */ + public function __destruct() { + unset($this->url, $this->method, $this->post_data, $this->headers, $this->options); + } +} + +/** + * RollingCurl custom exception + */ +class RollingCurlException extends Exception { +} + +/** + * Class that holds a rolling queue of curl requests. + * + * @throws RollingCurlException + */ +class RollingCurl { + /** + * @var int + * + * Window size is the max number of simultaneous connections allowed. + * + * REMEMBER TO RESPECT THE SERVERS: + * Sending too many requests at one time can easily be perceived + * as a DOS attack. Increase this window_size if you are making requests + * to multiple servers or have permission from the receving server admins. + */ + private $window_size = 5; + + /** + * @var float + * + * Timeout is the timeout used for curl_multi_select. + */ + private $timeout = 10; + + /** + * @var string|array + * + * Callback function to be applied to each result. + */ + private $callback; + + /** + * @var array + * + * Set your base options that you want to be used with EVERY request. + */ + protected $options = array( + CURLOPT_SSL_VERIFYPEER => 0, + CURLOPT_RETURNTRANSFER => 1, + CURLOPT_CONNECTTIMEOUT => 30, + CURLOPT_TIMEOUT => 30 + ); + + /** + * @var array + */ + private $headers = array(); + + /** + * @var Request[] + * + * The request queue + */ + private $requests = array(); + + /** + * @var RequestMap[] + * + * Maps handles to request indexes + */ + private $requestMap = array(); + + /** + * @param $callback + * Callback function to be applied to each result. + * + * Can be specified as 'my_callback_function' + * or array($object, 'my_callback_method'). + * + * Function should take three parameters: $response, $info, $request. + * $response is response body, $info is additional curl info. + * $request is the original request + * + * @return void + */ + function __construct($callback = null) { + $this->callback = $callback; + } + + /** + * @param string $name + * @return mixed + */ + public function __get($name) { + return (isset($this->{$name})) ? $this->{$name} : null; + } + + /** + * @param string $name + * @param mixed $value + * @return bool + */ + public function __set($name, $value) { + // append the base options & headers + if ($name == "options" || $name == "headers") { + $this->{$name} = $value + $this->{$name}; + } else { + $this->{$name} = $value; + } + return true; + } + + /** + * Add a request to the request queue + * + * @param Request $request + * @return bool + */ + public function add($request) { + $this->requests[] = $request; + return true; + } + + /** + * Create new Request and add it to the request queue + * + * @param string $url + * @param string $method + * @param $post_data + * @param $headers + * @param $options + * @return bool + */ + public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) { + $this->requests[] = new RollingCurlRequest($url, $method, $post_data, $headers, $options); + return true; + } + + /** + * Perform GET request + * + * @param string $url + * @param $headers + * @param $options + * @return bool + */ + public function get($url, $headers = null, $options = null) { + return $this->request($url, "GET", null, $headers, $options); + } + + /** + * Perform POST request + * + * @param string $url + * @param $post_data + * @param $headers + * @param $options + * @return bool + */ + public function post($url, $post_data = null, $headers = null, $options = null) { + return $this->request($url, "POST", $post_data, $headers, $options); + } + + /** + * Execute processing + * + * @param int $window_size Max number of simultaneous connections + * @return string|bool + */ + public function execute($window_size = null) { + // rolling curl window must always be greater than 1 + if (sizeof($this->requests) == 1) { + return $this->single_curl(); + } else { + // start the rolling curl. window_size is the max number of simultaneous connections + return $this->rolling_curl($window_size); + } + } + + /** + * Performs a single curl request + * + * @access private + * @return string + */ + private function single_curl() { + $ch = curl_init(); + $request = array_shift($this->requests); + $options = $this->get_options($request); + curl_setopt_array($ch, $options); + $output = curl_exec($ch); + $info = curl_getinfo($ch); + + // it's not neccesary to set a callback for one-off requests + if ($this->callback) { + $callback = $this->callback; + if (is_callable($this->callback)) { + call_user_func($callback, $output, $info, $request); + } + } + else + return $output; + return true; + } + + /** + * Performs multiple curl requests + * + * @access private + * @throws RollingCurlException + * @param int $window_size Max number of simultaneous connections + * @return bool + */ + private function rolling_curl($window_size = null) { + if ($window_size) + $this->window_size = $window_size; + + // make sure the rolling window isn't greater than the # of urls + if (sizeof($this->requests) < $this->window_size) + $this->window_size = sizeof($this->requests); + + if ($this->window_size < 2) { + throw new RollingCurlException("Window size must be greater than 1"); + } + + $master = curl_multi_init(); + + // start the first batch of requests + for ($i = 0; $i < $this->window_size; $i++) { + $ch = curl_init(); + + $options = $this->get_options($this->requests[$i]); + + curl_setopt_array($ch, $options); + curl_multi_add_handle($master, $ch); + + // Add to our request Maps + $key = (string) $ch; + $this->requestMap[$key] = $i; + } + + do { + while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM) ; + if ($execrun != CURLM_OK) + break; + // a request was just completed -- find out which one + while ($done = curl_multi_info_read($master)) { + + // get the info and content returned on the request + $info = curl_getinfo($done['handle']); + $output = curl_multi_getcontent($done['handle']); + + // send the return values to the callback function. + $callback = $this->callback; + if (is_callable($callback)) { + $key = (string) $done['handle']; + $request = $this->requests[$this->requestMap[$key]]; + unset($this->requestMap[$key]); + call_user_func($callback, $output, $info, $request); + } + + // start a new request (it's important to do this before removing the old one) + if ($i < sizeof($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) { + $ch = curl_init(); + $options = $this->get_options($this->requests[$i]); + curl_setopt_array($ch, $options); + curl_multi_add_handle($master, $ch); + + // Add to our request Maps + $key = (string) $ch; + $this->requestMap[$key] = $i; + $i++; + } + + // remove the curl handle that just completed + curl_multi_remove_handle($master, $done['handle']); + + } + + // Block for data in / output; error handling is done by curl_multi_exec + if ($running) + curl_multi_select($master, $this->timeout); + + } while ($running); + curl_multi_close($master); + return true; + } + + + /** + * Helper function to set up a new request by setting the appropriate options + * + * @access private + * @param Request $request + * @return array + */ + private function get_options($request) { + // options for this entire curl object + $options = $this->__get('options'); + if (ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) { + $options[CURLOPT_FOLLOWLOCATION] = 1; + $options[CURLOPT_MAXREDIRS] = 5; + } + $headers = $this->__get('headers'); + + // append custom options for this specific request + if ($request->options) { + $options = $request->options + $options; + } + + // set the request URL + $options[CURLOPT_URL] = $request->url; + + // posting data w/ this request? + if ($request->post_data) { + $options[CURLOPT_POST] = 1; + $options[CURLOPT_POSTFIELDS] = $request->post_data; + } + if ($headers) { + $options[CURLOPT_HEADER] = 0; + $options[CURLOPT_HTTPHEADER] = $headers; + } + + return $options; + } + + /** + * @return void + */ + public function __destruct() { + unset($this->window_size, $this->callback, $this->options, $this->headers, $this->requests); + } +} + --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/RollingCurlGroup.php.svn-base @@ -1,1 +1,218 @@ - +group = $group; + } + + /** + * Process the request + * + * + */ + function process($output, $info) { + if ($this->group) + $this->group->process($output, $info, $this); + } + + /** + * @return void + */ + public function __destruct() { + unset($this->group); + parent::__destruct(); + } + +} + +/** + * A group of curl requests. + * + * @throws RollingCurlGroupException * + */ +class RollingCurlGroup { + /** + * @var string group name + */ + protected $name; + + /** + * @var int total number of requests in a group + */ + protected $num_requests = 0; + + /** + * @var int total number of finished requests in a group + */ + protected $finished_requests = 0; + + /** + * @var array requests array + */ + private $requests = array(); + + /** + * @param string $name group name + * @return void + */ + function __construct($name) { + $this->name = $name; + } + + /** + * @return void + */ + public function __destruct() { + unset($this->name, $this->num_requests, $this->finished_requests, $this->requests); + } + + /** + * Adds request to a group + * + * @throws RollingCurlGroupException + * @param RollingCurlGroupRequest|array $request + * @return bool + */ + function add($request) { + if ($request instanceof RollingCurlGroupRequest) { + $request->setGroup($this); + $this->num_requests++; + $this->requests[] = $request; + } + else if (is_array($request)) { + foreach ($request as $req) + $this->add($req); + } + else + throw new RollingCurlGroupException("add: Request needs to be of instance RollingCurlGroupRequest"); + + return true; + } + + /** + * @throws RollingCurlGroupException + * @param RollingCurl $rc + * @return bool + */ + function addToRC(RollingCurl $rc){ + $ret = true; + + while (count($this->requests) > 0){ + $ret1 = $rc->add(array_shift($this->requests)); + if (!$ret1) + $ret = false; + } + + return $ret; + } + + /** + * Override to implement custom response processing. + * + * Don't forget to call parent::process(). + * + * @param string $output received page body + * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc. + * @param RollingCurlRequest $request request used + * @return void + */ + function process($output, $info, $request) { + $this->finished_requests++; + + if ($this->finished_requests >= $this->num_requests) + $this->finished(); + } + + /** + * Override to execute code after all requests in a group are processed. + * + * @return void + */ + function finished() { + } + +} + +/** + * Group version of rolling curl + */ +class GroupRollingCurl extends RollingCurl { + + /** + * @var mixed common callback for all groups + */ + private $group_callback = null; + + /** + * @param string $output received page body + * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc. + * @param RollingCurlRequest $request request used + * @return void + */ + protected function process($output, $info, $request) { + if ($request instanceof RollingCurlGroupRequest) + $request->process($output, $info); + + if (is_callable($this->group_callback)) + call_user_func($this->group_callback, $output, $info, $request); + } + + /** + * @param mixed $callback common callback for all groups + * @return void + */ + function __construct($callback = null) { + $this->group_callback = $callback; + + parent::__construct(array(&$this, "process")); + } + + /** + * Adds a group to processing queue + * + * @param RollingCurlGroup|Request $request + * @return bool + */ + public function add($request) { + if ($request instanceof RollingCurlGroup) + return $request->addToRC($this); + else + return parent::add($request); + } + + /** + * Execute processing + * + * @param int $window_size Max number of simultaneous connections + * @return bool|string + */ + public function execute($window_size = null) { + if (count($this->requests) == 0) + return false; + + return parent::execute($window_size); + } +} + --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/example.php.svn-base @@ -1,1 +1,66 @@ +(.*?)~i", $response, $out)) { + $title = $out[1]; + } + echo "$title
"; + print_r($info); + print_r($request); + echo "
"; +} + +// single curl request +$rc = new RollingCurl("request_callback"); +$rc->request("http://www.msn.com"); +$rc->execute(); + +// another single curl request +$rc = new RollingCurl("request_callback"); +$rc->request("http://www.google.com"); +$rc->execute(); + +echo "
"; + +// top 20 sites according to alexa (11/5/09) +$urls = array("http://www.google.com", + "http://www.facebook.com", + "http://www.yahoo.com", + "http://www.youtube.com", + "http://www.live.com", + "http://www.wikipedia.com", + "http://www.blogger.com", + "http://www.msn.com", + "http://www.baidu.com", + "http://www.yahoo.co.jp", + "http://www.myspace.com", + "http://www.qq.com", + "http://www.google.co.in", + "http://www.twitter.com", + "http://www.google.de", + "http://www.microsoft.com", + "http://www.google.cn", + "http://www.sina.com.cn", + "http://www.wordpress.com", + "http://www.google.co.uk"); + +$rc = new RollingCurl("request_callback"); +$rc->window_size = 20; +foreach ($urls as $url) { + $request = new RollingCurlRequest($url); + $rc->add($request); +} +$rc->execute(); + --- /dev/null +++ b/lib/rolling-curl/.svn/text-base/example_groups.php.svn-base @@ -1,1 +1,49 @@ +url . "\n"; + if ($this->test_verbose) + print_r($info); + + parent::process($output, $info); + } +} + +class TestCurlGroup extends RollingCurlGroup { + function process($output, $info, $request) { + echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n"; + parent::process($output, $info, $request); + } + + function finished() { + echo "Group CB: Finished" . $this->name . "\n"; + parent::finished(); + } +} + +$group = new TestCurlGroup("High"); +$group->add(new TestCurlRequest("www.google.de")); +$group->add(new TestCurlRequest("www.yahoo.de")); +$group->add(new TestCurlRequest("www.newyorktimes.com")); +$reqs[] = $group; + +$group = new TestCurlGroup("Normal"); +$group->add(new TestCurlRequest("twitter.com")); +$group->add(new TestCurlRequest("www.bing.com")); +$group->add(new TestCurlRequest("m.facebook.com")); +$reqs[] = $group; + +$reqs[] = new TestCurlRequest("www.kernel.org"); + +// No callback here, as its done in Request class +$rc = new GroupRollingCurl(); + +foreach ($reqs as $req) + $rc->add($req); + +$rc->execute(); --- /dev/null +++ b/lib/rolling-curl/CHANGELOG.txt @@ -1,1 +1,15 @@ +Rolling Curl changelog +====================== +September 13, 2010 +------------------ +- Bug #12, #14: Fixed default options overriding (LionsAd) +- Bug #10: Added use of curl_multi_select to avoid burning CPU (LionsAd) +- Enh #6, #9: Added $request as parameter to callback function (LionsAd) +- Chg: Request renamed to RollingCurlRequest (LionsAd) +- Added RollingCurlGroup class that allows processing groups of requests (LionsAd) +- More cleanup at unsetting a class (LionsAd) +- Timeout parameter for curl_multi_select is now configurable (LionsAd) +- single_curl now returns true (LionsAd) +- Readme corrections (Alexander Makarov) +- Code cleanup (Alexander Makarov) --- /dev/null +++ b/lib/rolling-curl/README.txt @@ -1,1 +1,210 @@ - +Rolling Curl +============ + +RollingCurl allows you to process multiple HTTP requests in parallel using CURL PHP library. + +Released under the Apache License 2.0. + +Authors +------- +- Was originally written by [Josh Fraser](joshfraser.com). +- Currently maintained by [Alexander Makarov](http://rmcreative.ru/). +- Received significant updates and patched from [LionsAd](http://github.com/LionsAd/rolling-curl). + +Overview +-------- +RollingCurl is a more efficient implementation of curl_multi() curl_multi is a great way to process multiple HTTP requests in parallel in PHP. +curl_multi is particularly handy when working with large data sets (like fetching thousands of RSS feeds at one time). Unfortunately there is +very little documentation on the best way to implement curl_multi. As a result, most of the examples around the web are either inefficient or +fail entirely when asked to handle more than a few hundred requests. + +The problem is that most implementations of curl_multi wait for each set of requests to complete before processing them. If there are too many requests +to process at once, they usually get broken into groups that are then processed one at a time. The problem with this is that each group has to wait for +the slowest request to download. In a group of 100 requests, all it takes is one slow one to delay the processing of 99 others. The larger the number of +requests you are dealing with, the more noticeable this latency becomes. + +The solution is to process each request as soon as it completes. This eliminates the wasted CPU cycles from busy waiting. Also there is a queue of +cURL requests to allow for maximum throughput. Each time a request is completed, a new one is added from the queue. By dynamically adding and removing +links, we keep a constant number of links downloading at all times. This gives us a way to throttle the amount of simultaneous requests we are sending. +The result is a faster and more efficient way of processing large quantities of cURL requests in parallel. + +Callbacks +--------- + +Each of requests usually do have a callback to process results that is being executed when request is done +(both successfully or not). + +Callback accepts three parameters and can look like the following one: +~~~ +[php] +function request_callback($response, $info, $request){ + // doing something with the data received +} +~~~ + +- $response contains received page body. +- $info is an associative array that holds various information about response such as HTTP response code, content type, +time taken to make request etc. +- $request contains RollingCurlRequest that was used to make request. + +Examples +-------- +### Hello world + +~~~ +[php] +// an array of URL's to fetch +$urls = array("http://www.google.com", + "http://www.facebook.com", + "http://www.yahoo.com"); + +// a function that will process the returned responses +function request_callback($response, $info, $request) { + // parse the page title out of the returned HTML + if (preg_match("~(.*?)~i", $response, $out)) { + $title = $out[1]; + } + echo "$title
"; + pr