From: maxious Date: Sun, 10 Apr 2011 10:57:41 +0000 Subject: Parallelise http transfers for trip planner tester X-Git-Url: http://maxious.lambdacomplex.org/git/?p=busui.git&a=commitdiff&h=94242a5095cccf5150d731e5102df0af306a8de5 --- Parallelise http transfers for trip planner tester --- --- /dev/null +++ b/aws/awsStartup.sh @@ -1,1 +1,37 @@ +#!/bin/bash +#this script should be run from a fresh git checkout from github +#ami base must have yum install lighttpd-fastcgi, git, tomcat6 +#screen php-cli php-gd tomcat6-webapps tomcat6-admin-webapps svn maven2 +#postgrtes postgres-server php-pg +#http://www.how2forge.org/installing-lighttpd-with-php5-and-mysql-support-on-fedora-12 +cp /root/aws.php /tmp/ +mkdir /var/www/lib/staticmaplite/cache +chcon -h system_u:object_r:httpd_sys_content_t /var/www +chcon -R -h root:object_r:httpd_sys_content_t /var/www/* +chcon -R -t httpd_sys_content_rw_t /var/www/lib/staticmaplite/cache +chmod -R 777 /var/www/lib/staticmaplite/cache +wget http://s3-ap-southeast-1.amazonaws.com/busresources/cbrfeed.zip \ +-O /var/www/cbrfeed.zip + +createdb transitdata +createlang -d transitdata plpgsql +psql -d transitdata -f /var/www/lib/postgis.sql +# curl https://github.com/maxious/ACTBus-ui/raw/master/transitdata.cbrfeed.sql.gz -o transitdata.cbrfeed.sql.gz +#made with pg_dump transitdata | gzip -c > transitdata.cbrfeed.sql.gz +gunzip /var/www/transitdata.cbrfeed.sql.gz +psql -d transitdata -f /var/www/transitdata.cbrfeed.sql +#createuser transitdata -SDRP +#password transitdata +#psql -d transitdata -c \"GRANT SELECT ON TABLE agency,calendar,calendar_dates,routes,stop_times,stops,trips TO transitdata;\" +php /var/www/updatedb.php + +wget http://s3-ap-southeast-1.amazonaws.com/busresources/Graph.obj \ +-O /tmp/Graph.obj +rm -rfv /usr/share/tomcat6/webapps/opentripplanner* +wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-webapp.war \ +-O /usr/share/tomcat6/webapps/opentripplanner-webapp.war +wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-api-webapp.war \ +-O /usr/share/tomcat6/webapps/opentripplanner-api-webapp.war +/etc/init.d/tomcat6 restart + --- /dev/null +++ b/aws/pg_hba.conf @@ -1,1 +1,75 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the +# PostgreSQL documentation for a complete description +# of this file. A short synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain socket, +# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an +# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", a database name, or +# a comma-separated list thereof. +# +# USER can be "all", a user name, a group name prefixed with "+", or +# a comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names from +# a separate file. +# +# CIDR-ADDRESS specifies the set of hosts the record matches. +# It is made up of an IP address and a CIDR mask that is an integer +# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies +# the number of significant bits in the mask. Alternatively, you can write +# an IP address and netmask in separate columns to specify the set of hosts. +# +# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", "krb5", +# "ident", "pam", "ldap" or "cert". Note that "password" sends passwords +# in clear text; "md5" is preferred since it sends encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different authentication +# methods - refer to the "Client Authentication" section in the documentation +# for a list of which options are available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other special +# characters must be quoted. Quoting one of the keywords "all", "sameuser" or +# "samerole" makes the name lose its special character, and just match a +# database or username with that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL listen +# on a non-local interface via the listen_addresses configuration parameter, +# or via the -i or -h command line switches. +# + + + +# TYPE DATABASE USER CIDR-ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust + --- /dev/null +++ b/include/common-db.inc.php @@ -1,1 +1,21 @@ + --- a/include/common-geo.inc.php +++ b/include/common-geo.inc.php @@ -70,6 +70,7 @@ else return round($km,2)."k"; } else return floor($km * 1000); } + function decodePolylineToArray($encoded) { // source: http://latlongeeks.com/forum/viewtopic.php?f=4&t=5 --- a/include/common-session.inc.php +++ b/include/common-session.inc.php @@ -17,7 +17,6 @@ } else { $geolocate = filter_var($_REQUEST['geolocate'], FILTER_SANITIZE_URL); - echo $_REQUEST['geolocate']; if (startsWith($geolocate, "-")) { $locateparts = explode(",", $geolocate); $_SESSION['lat'] = $locateparts[0]; --- a/include/db/route-dao.inc.php +++ b/include/db/route-dao.inc.php @@ -12,19 +12,7 @@ } function getRoutes() { global $conn; - $conditions = Array(); - if ($timingPointsOnly) $conditions[] = "substr(stop_code,1,2) != 'Wj'"; - if ($firstLetter != "") $conditions[] = "substr(stop_name,1,1) = '$firstLetter'"; - $query = "Select * from routes"; - if (sizeof($conditions) > 0) { - if (sizeof($conditions) > 1) { - $query .= " Where ".implode(" AND ",$conditions)." "; - } - else { - $query .= " Where ".$conditions[0]." "; - } - } - $query .= " order by route_short_name;"; + $query = "Select * from routes order by route_short_name;"; debug($query,"database"); $result = pg_query($conn, $query); if (!$result) { @@ -34,9 +22,14 @@ return pg_fetch_all($result); } -function findRouteByNumber($routeNumber) { +function getRoutesByNumber($routeNumber = "") { global $conn; - $query = "Select * from routes where route_short_name = '$routeNumber';"; + if ($routeNumber != "") { + $query = "Select distinct routes.route_id,routes.route_short_name,routes.route_long_name,service_id from routes join trips on trips.route_id = +routes.route_id join stop_times on stop_times.trip_id = trips.trip_id where route_short_name = '$routeNumber' order by route_short_name;"; + } else { + $query = "SELECT DISTINCT route_short_name from routes order by route_short_name"; + } debug($query,"database"); $result = pg_query($conn, $query); if (!$result) { @@ -47,6 +40,7 @@ } function getRouteNextTrip($routeID) { + global $conn; $query = "select * from routes join trips on trips.route_id = routes.route_id join stop_times on stop_times.trip_id = trips.trip_id where arrival_time > CURRENT_TIME and routes.route_id = '$routeID' order by @@ -57,7 +51,83 @@ databaseError(pg_result_error($result)); return Array(); } - return pg_fetch_assoc($result); /* + return pg_fetch_assoc($result); } +function getRouteTrips($routeID) { + global $conn; + $query = "select * from routes join trips on trips.route_id = routes.route_id +join stop_times on stop_times.trip_id = trips.trip_id where routes.route_id = '$routeID' order by +arrival_time "; + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); + } +function getRoutesByDestination($destination = "", $service_period = "") { + global $conn; + if ($service_period == "") $service_period = service_period(); + if ($destination != "") { + $query = "SELECT DISTINCT trips.route_id,route_short_name,route_long_name, service_id +FROM stop_times join trips on trips.trip_id = +stop_times.trip_id join routes on trips.route_id = routes.route_id +WHERE route_long_name = '$destination' AND service_id='$service_period' order by route_short_name"; + } else { + $query = "SELECT DISTINCT route_long_name +FROM stop_times join trips on trips.trip_id = +stop_times.trip_id join routes on trips.route_id = routes.route_id +WHERE service_id='$service_period' order by route_long_name"; + } + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); +} +function getRoutesBySuburb($suburb, $service_period = "") { + if ($service_period == "") $service_period = service_period(); + global $conn; + $query = "SELECT DISTINCT service_id,trips.route_id,route_short_name,route_long_name +FROM stop_times join trips on trips.trip_id = stop_times.trip_id +join routes on trips.route_id = routes.route_id +join stops on stops.stop_id = stop_times.stop_id +WHERE zone_id LIKE '%$suburb;%' AND service_id='$service_period' ORDER BY route_short_name"; + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); +} + +function getRoutesNearby($lat, $lng, $limit = "", $distance = 500) { + + + if ($service_period == "") $service_period = service_period(); + if ($limit != "") $limit = " LIMIT $limit "; + global $conn; + $query = "SELECT service_id,trips.route_id,route_short_name,route_long_name, + min(ST_Distance(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), FALSE)) as distance +FROM stop_times +join trips on trips.trip_id = stop_times.trip_id +join routes on trips.route_id = routes.route_id +join stops on stops.stop_id = stop_times.stop_id +WHERE service_id='$service_period' +AND ST_DWithin(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), $distance, FALSE) + group by service_id,trips.route_id,route_short_name,route_long_name + order by distance $limit"; + debug($query,"database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); +} ?> --- a/include/db/stop-dao.inc.php +++ b/include/db/stop-dao.inc.php @@ -1,33 +1,33 @@ 0) { - if (sizeof($conditions) > 1) { - $query .= " Where ".implode(" AND ",$conditions)." "; - } - else { - $query .= " Where ".$conditions[0]." "; - } - } - $query .= " order by stop_name;"; - debug($query,"database"); + if (sizeof($conditions) > 0) { + if (sizeof($conditions) > 1) { + $query.= " Where " . implode(" AND ", $conditions) . " "; + } + else { + $query.= " Where " . $conditions[0] . " "; + } + } + $query.= " order by stop_name;"; + debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { databaseError(pg_result_error($result)); @@ -35,14 +35,15 @@ } return pg_fetch_all($result); } -function getNearbyStops($lat, $lng, $limit, $distance = 1000) +function getNearbyStops($lat, $lng, $limit = "", $distance = 1000) { - if ($lat == null || $lng == null) return Array(); - global $conn; - $query = "Select *, ST_Distance(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), FALSE) as distance + if ($lat == null || $lng == null) return Array(); + if ($limit != "") $limit = " LIMIT $limit "; + global $conn; + $query = "Select *, ST_Distance(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), FALSE) as distance from stops WHERE ST_DWithin(position, ST_GeographyFromText('SRID=4326;POINT($lng $lat)'), $distance, FALSE) - order by distance;"; - debug($query,"database"); + order by distance $limit;"; + debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { databaseError(pg_result_error($result)); @@ -52,9 +53,9 @@ } function getStopsBySuburb($suburb) { -global $conn; - $query = "Select * from stops where zone_id LIKE '%$suburb;%' order by stop_name;"; - debug($query,"database"); + global $conn; + $query = "Select * from stops where zone_id LIKE '%$suburb;%' order by stop_name;"; + debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { databaseError(pg_result_error($result)); @@ -64,58 +65,79 @@ } function getStopRoutes($stopID, $service_period) { - if ($service_period == "") $service_period = service_period(); - global $conn; - $query = "SELECT service_id,trips.route_id,route_short_name,route_long_name + if ($service_period == "") $service_period = service_period(); + global $conn; + $query = "SELECT service_id,trips.route_id,route_short_name,route_long_name FROM stop_times join trips on trips.trip_id = stop_times.trip_id join routes on trips.route_id = routes.route_id WHERE stop_id = '$stopID' AND service_id='$service_period'"; - debug($query,"database"); - $result = pg_query($conn, $query); - if (!$result) { - databaseError(pg_result_error($result)); - return Array(); - } - return pg_fetch_all($result);} -function getStopTrips($stopID, $service_period = "") -{ - if ($service_period == "") $service_period = service_period(); - global $conn; - $query = "SELECT stop_times.trip_id,arrival_time,stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name -FROM stop_times join trips on trips.trip_id = -stop_times.trip_id join routes on trips.route_id = routes.route_id WHERE stop_id = '$stopID' AND service_id='$service_period'"; - debug($query,"database"); + debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { databaseError(pg_result_error($result)); return Array(); } return pg_fetch_all($result); - +} +function getStopTrips($stopID, $service_period = "", $afterTime = "") +{ + if ($service_period == "") $service_period = service_period(); + $afterCondition = "AND arrival_time > '$afterTime'"; + global $conn; + if ($afterTime != "") { + $query = " SELECT stop_times.trip_id,stop_times.arrival_time,stop_times.stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name, start_times.arrival_time as start_time +FROM stop_times +join trips on trips.trip_id = +stop_times.trip_id +join routes on trips.route_id = routes.route_id , (SELECT trip_id,arrival_time from stop_times + WHERE stop_times.arrival_time IS NOT NULL + AND stop_sequence = '1') as start_times +WHERE stop_times.stop_id = '$stopID' +AND stop_times.trip_id = start_times.trip_id +AND service_id='$service_period' +AND start_times.arrival_time > '$afterTime' +ORDER BY start_time"; + } + else { + $query = "SELECT stop_times.trip_id,arrival_time,stop_times.stop_id,stop_sequence,service_id,trips.route_id,route_short_name,route_long_name +FROM stop_times +join trips on trips.trip_id = +stop_times.trip_id +join routes on trips.route_id = routes.route_id +WHERE stop_times.stop_id = '$stopID' +AND service_id='$service_period' +ORDER BY arrival_time"; + } + debug($query, "database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + return pg_fetch_all($result); } function getStopTripsWithTimes($stopID, $time = "", $service_period = "", $time_range = "") { - if ($service_period == "") $service_period = service_period(); - if ($time_range == "") $time_range = (24*60*60); - if ($time == "") $time = ($_SESSION['time'] ? $_SESSION['time'] : date("h:i:00")); - $trips = getStopTrips($stopID,$service_period); - $timedTrips = Array(); - foreach ($trips as $trip) { - if ($trip['arrival_time'] != "") { - if (strtotime($trip['arrival_time']) > strtotime($time) and strtotime($trip['arrival_time']) < (strtotime($time) + $time_range)){ - $timedTrips[] = $trip; - } - } else { - $tripstarttime = getTripStartTime($trip['trip_id']); - if ($tripstarttime > $time and $tripstarttime < ($time + $time_range)) { - $timedtrip = getTimeInterpolatedTripStop($trip['trip_id'], $trip['stop_sequence']); - if (strtotime($timedtrip['arrival_time']) > strtotime($time) and $timedtrip['arrival_time'] < (strtotime($time) + strtotime($time_range))){ - $timedTrips[] = $timedTrip; - } - } - } - if (sizeof($timedTrips) > limit) break; - } - sktimesort($timedTrips,"arrival_time", true); - return $timedTrips; + if ($service_period == "") $service_period = service_period(); + if ($time_range == "") $time_range = (24 * 60 * 60); + if ($time == "") $time = ($_SESSION['time'] ? $_SESSION['time'] : date("H:i:s")); + if ($limit == "") $limit = 10; + $trips = getStopTrips($stopID, $service_period, $time); + $timedTrips = Array(); + foreach ($trips as $trip) { + if ($trip['arrival_time'] != "") { + if (strtotime($trip['arrival_time']) > strtotime($time) and strtotime($trip['arrival_time']) < (strtotime($time) + $time_range)) { + $timedTrips[] = $trip; + } + } + else { + $timedTrip = getTimeInterpolatedTripAtStop($trip['trip_id'], $trip['stop_sequence']); + if ($timedTrip['arrival_time'] > $time and strtotime($timedTrip['arrival_time']) < (strtotime($time) + $time_range)) { + $timedTrips[] = $timedTrip; + } + } + if (sizeof($timedTrips) > $limit) break; + } + sktimesort($timedTrips, "arrival_time", true); + return $timedTrips; } ?> --- a/include/db/trip-dao.inc.php +++ b/include/db/trip-dao.inc.php @@ -1,109 +1,146 @@ $stopTime) { + if ($stopTime['arrival_time'] != "") { + // is timepoint + $cur_timepoint = $stopTime; + $distance_between_timepoints = 0.0; + $distance_traveled_between_timepoints = 0.0; + if ($i + 1 < sizeof($stopTimes)) { + $k = $i + 1; + $distance_between_timepoints += distance($stopTimes[$k - 1]["stop_lat"], $stopTimes[$k - 1]["stop_lon"], $stopTimes[$k]["stop_lat"], $stopTimes[$k]["stop_lon"]); + while ($stopTimes[$k]["arrival_time"] == "" && $k + 1 < sizeof($stopTimes)) { + $k += 1; + //echo "k".$k; + $distance_between_timepoints += distance($stopTimes[$k - 1]["stop_lat"], $stopTimes[$k - 1]["stop_lon"], $stopTimes[$k]["stop_lat"], $stopTimes[$k]["stop_lon"]); + } + $next_timepoint = $stopTimes[$k]; + $rv[] = $stopTime; + } + } + else { + // is untimed point + //echo "i".$i; + $distance_traveled_between_timepoints += distance($stopTimes[$i - 1]["stop_lat"], $stopTimes[$i - 1]["stop_lon"], $stopTimes[$i]["stop_lat"], $stopTimes[$i]["stop_lon"]); + //echo "$distance_traveled_between_timepoints / $distance_between_timepoints
"; + $distance_percent = $distance_traveled_between_timepoints / $distance_between_timepoints; + if ($next_timepoint["arrival_time"] != "") { + $total_time = strtotime($next_timepoint["arrival_time"]) - strtotime($cur_timepoint["arrival_time"]); + //echo strtotime($next_timepoint["arrival_time"])." - ".strtotime($cur_timepoint["arrival_time"])."
"; + $time_estimate = ($distance_percent * $total_time) + strtotime($cur_timepoint["arrival_time"]); + $stopTime["arrival_time"] = date("H:i:s", $time_estimate); + } else { + $stopTime["arrival_time"] = $cur_timepoint["arrival_time"]; + } + $rv[] = $stopTime; + //var_dump($rv); + } + } + return $rv; } - -function getTripStartTime($tripID) { - $query = 'SELECT arrival_secs,departure_secs FROM stop_times WHERE trip_id=? ORDER BY stop_sequence LIMIT 1'; - +function getTimeInterpolatedTripAtStop($tripID, $stop_sequence) +{ + foreach (getTimeInterpolatedTrip($tripID) as $tripStop) { + if ($tripStop['stop_sequence'] == $stop_sequence) return $tripStop; + } + return Array(); } - -function viaPointNames($tripid, $stopid) +function getTripStartTime($tripID) { - global $conn; - $query = "SELECT stop_name + global $conn; + $query = "Select * from stop_times + where trip_id = '$tripID' + AND arrival_time IS NOT NULL + AND stop_sequence = '1'"; + debug($query, "database"); + $result = pg_query($conn, $query); + if (!$result) { + databaseError(pg_result_error($result)); + return Array(); + } + $r = pg_fetch_assoc($result); + return $r['arrival_time']; +} +function viaPointNames($tripid, $stop_sequence = "") +{ + global $conn; + $query = "SELECT stop_name FROM stop_times join stops on stops.stop_id = stop_times.stop_id WHERE stop_times.trip_id = '$tripid' -AND stop_sequence > '$stop_sequence' -AND substr(stop_code,1,2) != 'Wj' ORDER BY stop_sequence"; - debug($query,"database"); +".($stop_sequence != "" ? "AND stop_sequence > '$stop_sequence'" : ""). +"AND substr(stop_code,1,2) != 'Wj' ORDER BY stop_sequence"; + debug($query, "database"); $result = pg_query($conn, $query); if (!$result) { databaseError(pg_result_error($result)); --- a/labs/tripPlannerTester.kml.php +++ b/labs/tripPlannerTester.kml.php @@ -8,6 +8,50 @@ } else { return (($pBegin - $pEnd) * (1 - ($pStep / $pMax))) + $pEnd; + } +} +require ("../lib/rolling-curl/RollingCurl.php"); +function processResult_cb($response, $info, $request) +{ + global $testRegions, $regionTimes,$csv,$kml, $latdeltasize,$londeltasize; + $md = $request->metadata; + $tripplan = json_decode($response); + $plans = Array(); + //var_dump(Array($info, $request)); + if (is_array($tripplan->plan->itineraries->itinerary)) { + foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) { + $plans[floor($itinerary->duration / 60000) ] = $itinerary; + } + } + else { + $plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary; + } + if ($csv) echo "{$md['i']},{$md['j']}," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,{$md['key']}\n"; + if ($kml) { + $time = min(array_keys($plans)); + $plan = ""; + if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) { + foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) { + $plan.= processLeg($legNumber, $leg) . ","; + } + } + else { + $plan.= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg); + } + if (isset($tripplan->error) && $tripplan->error->id == 404) { + $time = 999; + $plan = "Trip not possible without excessive walking from nearest bus stop"; + } + $testRegions[] = Array( + "lat" => $md['i'], + "lon" => $md['j'], + "time" => $time, + "latdeltasize" => $latdeltasize, + "londeltasize" => $londeltasize, + "regionname" => $md['key'], + "plan" => $plan . "
original plan" + ); + $regionTimes[] = $time; } } function Gradient($HexFrom, $HexTo, $ColorSteps) @@ -49,18 +93,17 @@ //} //$walkingstep.= floor($step->distance) . "m"; //return $walkingstep; + } } $csv = false; $kml = true; if ($kml) { - //header('Content-Type: application/vnd.google-earth.kml+xml'); + header('Content-Type: application/vnd.google-earth.kml+xml'); echo ' '; } include ('../include/common.inc.php'); -//Test code to grab transit times -// make sure to sleep(10); $boundingBoxes = Array( "belconnen" => Array( "startlat" => - 35.1928, @@ -105,72 +148,29 @@ $useragent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1"; if ($csv) echo "
";
 if ($csv) echo "lat,lon,time,latdeltasize, londeltasize, region key name\n";
+$rc = new RollingCurl("processResult_cb");
+$rc->window_size = 3;
 foreach ($boundingBoxes as $key => $boundingBox) {
 	for ($i = $boundingBox['startlat']; $i >= $boundingBox['finishlat']; $i-= $latdeltasize) {
 		for ($j = $boundingBox['startlon']; $j <= $boundingBox['finishlon']; $j+= $londeltasize) {
 			$url = $otpAPIurl . "ws/plan?date=" . urlencode($startDate) . "&time=" . urlencode($startTime) . "&mode=TRANSIT%2CWALK&optimize=QUICK&maxWalkDistance=440&wheelchair=false&toPlace=" . $i . "," . $j . "&fromPlace=$fromPlace";
-			$ch = curl_init($url);
-			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
-			curl_setopt($ch, CURLOPT_HEADER, 0);
-			curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+			$request = new RollingCurlRequest($url);
+			$request->headers = Array(
 				"Accept: application/json"
-			));
-			curl_setopt($ch, CURLOPT_TIMEOUT, 5);
-			$page = curl_exec($ch);
-			if (curl_errno($ch)) {
-				if ($csv) echo "Trip planner temporarily unavailable: " . curl_errno($ch) . " " . curl_error($ch);
-			}
-			else {
-				$tripplan = json_decode($page); 
-				$plans = Array();
-				if (is_array($tripplan->plan->itineraries->itinerary)) {
-					foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) {
-						$plans[floor($itinerary->duration / 60000) ] = $itinerary;
-					}
-				}
-				else {
-					$plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary;
-				}
-				if ($csv) echo "$i,$j," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,$key\n";
-				if ($kml) {
-					$time = min(array_keys($plans));
-					$plan = "";
-					if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) {
-						foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) {
-							$plan .= processLeg($legNumber, $leg).",";
-						}
-					}
-					else {
-						$plan .= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg);
-					}
-						if (isset($tripplan->error) && $tripplan->error->id == 404) {
-							$time = 999;
-							$plan = "Trip not possible without excessive walking from nearest bus stop";
-						}
-					$testRegions[] = Array(
-						"lat" => $i,
-						"lon" => $j,
-						"time" => $time,
-						"latdeltasize" => $latdeltasize,
-						"londeltasize" => $londeltasize,
-						"regionname" => $key,
-						"plan" => $plan . "
original plan"
-					);
-					$regionTimes[] = $time;
-				}
-			}
-			flush(); @ob_flush();
-			curl_close($ch);
-		}
-	}
-}
+			);
+			$request->metadata = Array( "i" => $i, "j" => $j, "key" => $key);
+			$rc->add($request);
+		}
+	}
+}
+$rc->execute();
 if ($kml) {
 	$colorSteps = 9;
 	//$minTime = min($regionTimes);
 	//$maxTime = max($regionTimes);
 	//$rangeTime = $maxTime - $minTime;
 	//$deltaTime = $rangeTime / $colorSteps;
-	$Gradients = Gradient(strrev("66FF00"), strrev("FF0000"), $colorSteps); // KML is BGR not RGB so strrev
+	$Gradients = Gradient(strrev("66FF00") , strrev("FF0000") , $colorSteps); // KML is BGR not RGB so strrev
 	foreach ($testRegions as $testRegion) {
 		//$band = (floor(($testRegion[time] - $minTime) / $deltaTime));
 		$band = (floor($testRegion[time] / 10));

--- /dev/null
+++ b/lib/rolling-curl/.svn/all-wcprops
@@ -1,1 +1,42 @@
+K 25
+svn:wc:ra_dav:version-url
+V 22
+/svn/!svn/ver/20/trunk
+END
+RollingCurlGroup.php
+K 25
+svn:wc:ra_dav:version-url
+V 43
+/svn/!svn/ver/20/trunk/RollingCurlGroup.php
+END
+example_groups.php
+K 25
+svn:wc:ra_dav:version-url
+V 41
+/svn/!svn/ver/20/trunk/example_groups.php
+END
+example.php
+K 25
+svn:wc:ra_dav:version-url
+V 34
+/svn/!svn/ver/20/trunk/example.php
+END
+RollingCurl.php
+K 25
+svn:wc:ra_dav:version-url
+V 38
+/svn/!svn/ver/20/trunk/RollingCurl.php
+END
+CHANGELOG.txt
+K 25
+svn:wc:ra_dav:version-url
+V 36
+/svn/!svn/ver/20/trunk/CHANGELOG.txt
+END
+README.txt
+K 25
+svn:wc:ra_dav:version-url
+V 33
+/svn/!svn/ver/20/trunk/README.txt
+END
 

--- /dev/null
+++ b/lib/rolling-curl/.svn/entries
@@ -1,1 +1,233 @@
-
+10
+
+dir
+20
+http://rolling-curl.googlecode.com/svn/trunk
+http://rolling-curl.googlecode.com/svn
+
+
+
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+74aa2acc-2e27-11de-b2a4-4f96ceaaac44
+
+RollingCurlGroup.php
+file
+
+
+
+
+2011-04-10T08:32:48.081650Z
+73c08d9e9e24b4adc89816624c7aca30
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+5152
+
+example_groups.php
+file
+
+
+
+
+2011-04-10T08:32:48.082650Z
+907ed82a47d346c39acbd5578e1d0230
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1367
+
+example.php
+file
+
+
+
+
+2011-04-10T08:32:48.083650Z
+87aa845abfaffc09ed4eca024f2a8b8a
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1860
+
+RollingCurl.php
+file
+
+
+
+
+2011-04-10T08:32:48.084650Z
+205391c449f3f3ee050004dadc374dc8
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+10444
+
+CHANGELOG.txt
+file
+
+
+
+
+2011-04-10T08:32:48.085650Z
+d0452f6f9530ed04580159121d0fd5f7
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+662
+
+README.txt
+file
+
+
+
+
+2011-04-10T08:32:48.085650Z
+60dd357081431c0f2b82989cdbce8615
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+6355
+
+

--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/CHANGELOG.txt.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
 

--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/RollingCurlGroup.php.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
 

--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/example_groups.php.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
 

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/CHANGELOG.txt.svn-base
@@ -1,1 +1,15 @@
+Rolling Curl changelog
+======================
 
+September 13, 2010
+------------------
+- Bug #12, #14: Fixed default options overriding (LionsAd)
+- Bug #10: Added use of curl_multi_select to avoid burning CPU (LionsAd)
+- Enh #6, #9: Added $request as parameter to callback function (LionsAd)
+- Chg: Request renamed to RollingCurlRequest (LionsAd)
+- Added RollingCurlGroup class that allows processing groups of requests (LionsAd)
+- More cleanup at unsetting a class (LionsAd)
+- Timeout parameter for curl_multi_select is now configurable (LionsAd)
+- single_curl now returns true (LionsAd)
+- Readme corrections (Alexander Makarov)
+- Code cleanup (Alexander Makarov)

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/README.txt.svn-base
@@ -1,1 +1,210 @@
-
+Rolling Curl
+============
+
+RollingCurl allows you to process multiple HTTP requests in parallel using CURL PHP library.
+
+Released under the Apache License 2.0.
+
+Authors
+-------
+- Was originally written by [Josh Fraser](joshfraser.com).
+- Currently maintained by [Alexander Makarov](http://rmcreative.ru/).
+- Received significant updates and patched from [LionsAd](http://github.com/LionsAd/rolling-curl).
+
+Overview
+--------
+RollingCurl is a more efficient implementation of curl_multi() curl_multi is a great way to process multiple HTTP requests in parallel in PHP.
+curl_multi is particularly handy when working with large data sets (like fetching thousands of RSS feeds at one time). Unfortunately there is
+very little documentation on the best way to implement curl_multi. As a result, most of the examples around the web are either inefficient or
+fail entirely when asked to handle more than a few hundred requests.
+
+The problem is that most implementations of curl_multi wait for each set of requests to complete before processing them. If there are too many requests
+to process at once, they usually get broken into groups that are then processed one at a time. The problem with this is that each group has to wait for
+the slowest request to download. In a group of 100 requests, all it takes is one slow one to delay the processing of 99 others. The larger the number of
+requests you are dealing with, the more noticeable this latency becomes.
+
+The solution is to process each request as soon as it completes. This eliminates the wasted CPU cycles from busy waiting. Also there is a queue of
+cURL requests to allow for maximum throughput. Each time a request is completed, a new one is added from the queue. By dynamically adding and removing
+links, we keep a constant number of links downloading at all times. This gives us a way to throttle the amount of simultaneous requests we are sending.
+The result is a faster and more efficient way of processing large quantities of cURL requests in parallel.
+
+Callbacks
+---------
+
+Each of requests usually do have a callback to process results that is being executed when request is done
+(both successfully or not).
+
+Callback accepts three parameters and can look like the following one:
+~~~
+[php]
+function request_callback($response, $info, $request){
+    // doing something with the data received
+}
+~~~
+
+- $response contains received page body.
+- $info is an associative array that holds various information about response such as HTTP response code, content type,
+time taken to make request etc.
+- $request contains RollingCurlRequest that was used to make request.
+
+Examples
+--------
+### Hello world
+
+~~~
+[php]
+// an array of URL's to fetch
+$urls = array("http://www.google.com",
+              "http://www.facebook.com",
+              "http://www.yahoo.com");
+
+// a function that will process the returned responses
+function request_callback($response, $info, $request) {
+	// parse the page title out of the returned HTML
+	if (preg_match("~(.*?)~i", $response, $out)) {
+		$title = $out[1];
+	}
+	echo "$title
";
+	print_r($info);
+	echo "
";
+}
+
+// create a new RollingCurl object and pass it the name of your custom callback function
+$rc = new RollingCurl("request_callback");
+// the window size determines how many simultaneous requests to allow.
+$rc->window_size = 20;
+foreach ($urls as $url) {
+    // add each request to the RollingCurl object
+    $request = new RollingCurlRequest($url);
+    $rc->add($request);
+}
+$rc->execute();
+~~~
+
+
+### Setting custom options
+
+Set custom options for EVERY request:
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$rc->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
+$rc->execute();
+~~~
+
+Set custom options for A SINGLE request:
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$request = new RollingCurlRequest($url);
+$request->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
+$rc->add($request);
+$rc->execute();
+~~~
+
+### Shortcuts
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$rc->get("http://www.google.com");
+$rc->get("http://www.yahoo.com");
+$rc->execute();
+~~~
+
+### Class callbacks
+
+~~~
+[php]
+class MyInfoCollector {
+    private $rc;
+
+    function __construct(){
+        $this->rc = new RollingCurl(array($this, 'processPage'));
+    }
+
+    function processPage($response, $info, $request){
+      //...
+    }
+
+    function run($urls){
+        foreach ($urls as $url){
+            $request = new RollingCurlRequest($url);
+            $this->rc->add($request);
+        }
+        $this->rc->execute();
+    }
+}
+
+$collector = new MyInfoCollector();
+$collector->run(array(
+    'http://google.com/',
+    'http://yahoo.com/'
+));
+~~~
+
+### Using RollingCurlGroup
+
+~~~
+[php]
+class TestCurlRequest extends RollingCurlGroupRequest {
+    public $test_verbose = true;
+
+    function process($output, $info) {
+        echo "Processing " . $this->url . "\n";
+        if ($this->test_verbose)
+            print_r($info);
+
+        parent::process($output, $info);
+    }
+}
+
+class TestCurlGroup extends RollingCurlGroup {
+    function process($output, $info, $request) {
+        echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n";
+        parent::process($output, $info, $request);
+    }
+
+    function finished() {
+        echo "Group CB: Finished" . $this->name . "\n";
+        parent::finished();
+    }
+}
+
+$group = new TestCurlGroup("High");
+$group->add(new TestCurlRequest("www.google.de"));
+$group->add(new TestCurlRequest("www.yahoo.de"));
+$group->add(new TestCurlRequest("www.newyorktimes.com"));
+$reqs[] = $group;
+
+$group = new TestCurlGroup("Normal");
+$group->add(new TestCurlRequest("twitter.com"));
+$group->add(new TestCurlRequest("www.bing.com"));
+$group->add(new TestCurlRequest("m.facebook.com"));
+$reqs[] = $group;
+
+$reqs[] = new TestCurlRequest("www.kernel.org");
+
+// No callback here, as its done in Request class
+$rc = new GroupRollingCurl();
+
+foreach ($reqs as $req)
+$rc->add($req);
+
+$rc->execute();
+~~~
+
+The same function (add) can be used both for adding requests and groups of requests.
+The "callback" in request and groups is:
+
+process($output, $info)
+
+and
+
+process($output, $info, $request)
+
+Also you can override RollingCurlGroup::finished() that will be executed right after finishing group processing.
+
+$Id$

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/RollingCurl.php.svn-base
@@ -1,1 +1,375 @@
-
+url = $url;
+        $this->method = $method;
+        $this->post_data = $post_data;
+        $this->headers = $headers;
+        $this->options = $options;
+    }
+
+    /**
+     * @return void
+     */
+    public function __destruct() {
+        unset($this->url, $this->method, $this->post_data, $this->headers, $this->options);
+    }
+}
+
+/**
+ * RollingCurl custom exception
+ */
+class RollingCurlException extends Exception {
+}
+
+/**
+ * Class that holds a rolling queue of curl requests.
+ *
+ * @throws RollingCurlException
+ */
+class RollingCurl {
+    /**
+     * @var int
+     *
+     * Window size is the max number of simultaneous connections allowed.
+     *
+     * REMEMBER TO RESPECT THE SERVERS:
+     * Sending too many requests at one time can easily be perceived
+     * as a DOS attack. Increase this window_size if you are making requests
+     * to multiple servers or have permission from the receving server admins.
+     */
+    private $window_size = 5;
+
+    /**
+     * @var float
+     *
+     * Timeout is the timeout used for curl_multi_select.
+     */
+    private $timeout = 10;
+
+    /**
+     * @var string|array
+     *
+     * Callback function to be applied to each result.
+     */
+    private $callback;
+
+    /**
+     * @var array
+     *
+     * Set your base options that you want to be used with EVERY request.
+     */
+    protected $options = array(
+        CURLOPT_SSL_VERIFYPEER => 0,
+        CURLOPT_RETURNTRANSFER => 1,
+        CURLOPT_CONNECTTIMEOUT => 30,
+        CURLOPT_TIMEOUT => 30
+    );
+
+    /**
+     * @var array
+     */
+    private $headers = array();
+
+    /**
+     * @var Request[]
+     *
+     * The request queue
+     */
+    private $requests = array();
+
+    /**
+     * @var RequestMap[]
+     *
+     * Maps handles to request indexes
+     */
+    private $requestMap = array();
+
+    /**
+     * @param  $callback
+     * Callback function to be applied to each result.
+     *
+     * Can be specified as 'my_callback_function'
+     * or array($object, 'my_callback_method').
+     *
+     * Function should take three parameters: $response, $info, $request.
+     * $response is response body, $info is additional curl info.
+     * $request is the original request
+     *
+     * @return void
+     */
+    function __construct($callback = null) {
+        $this->callback = $callback;
+    }
+
+    /**
+     * @param string $name
+     * @return mixed
+     */
+    public function __get($name) {
+        return (isset($this->{$name})) ? $this->{$name} : null;
+    }
+
+    /**
+     * @param string $name
+     * @param mixed $value
+     * @return bool
+     */
+    public function __set($name, $value) {
+        // append the base options & headers
+        if ($name == "options" || $name == "headers") {
+            $this->{$name} = $value + $this->{$name};
+        } else {
+            $this->{$name} = $value;
+        }
+        return true;
+    }
+
+    /**
+     * Add a request to the request queue
+     *
+     * @param Request $request
+     * @return bool
+     */
+    public function add($request) {
+        $this->requests[] = $request;
+        return true;
+    }
+
+    /**
+     * Create new Request and add it to the request queue
+     *
+     * @param string $url
+     * @param string $method
+     * @param  $post_data
+     * @param  $headers
+     * @param  $options
+     * @return bool
+     */
+    public function request($url, $method = "GET", $post_data = null, $headers = null, $options = null) {
+        $this->requests[] = new RollingCurlRequest($url, $method, $post_data, $headers, $options);
+        return true;
+    }
+
+    /**
+     * Perform GET request
+     *
+     * @param string $url
+     * @param  $headers
+     * @param  $options
+     * @return bool
+     */
+    public function get($url, $headers = null, $options = null) {
+        return $this->request($url, "GET", null, $headers, $options);
+    }
+
+    /**
+     * Perform POST request
+     *
+     * @param string $url
+     * @param  $post_data
+     * @param  $headers
+     * @param  $options
+     * @return bool
+     */
+    public function post($url, $post_data = null, $headers = null, $options = null) {
+        return $this->request($url, "POST", $post_data, $headers, $options);
+    }
+
+    /**
+     * Execute processing
+     *
+     * @param int $window_size Max number of simultaneous connections
+     * @return string|bool
+     */
+    public function execute($window_size = null) {
+        // rolling curl window must always be greater than 1
+        if (sizeof($this->requests) == 1) {
+            return $this->single_curl();
+        } else {
+            // start the rolling curl. window_size is the max number of simultaneous connections
+            return $this->rolling_curl($window_size);
+        }
+    }
+
+    /**
+     * Performs a single curl request
+     *
+     * @access private
+     * @return string
+     */
+    private function single_curl() {
+        $ch = curl_init();
+        $request = array_shift($this->requests);
+        $options = $this->get_options($request);
+        curl_setopt_array($ch, $options);
+        $output = curl_exec($ch);
+        $info = curl_getinfo($ch);
+
+        // it's not neccesary to set a callback for one-off requests
+        if ($this->callback) {
+            $callback = $this->callback;
+            if (is_callable($this->callback)) {
+                call_user_func($callback, $output, $info, $request);
+            }
+        }
+        else
+            return $output;
+        return true;
+    }
+
+    /**
+     * Performs multiple curl requests
+     *
+     * @access private
+     * @throws RollingCurlException
+     * @param int $window_size Max number of simultaneous connections
+     * @return bool
+     */
+    private function rolling_curl($window_size = null) {
+        if ($window_size)
+            $this->window_size = $window_size;
+
+        // make sure the rolling window isn't greater than the # of urls
+        if (sizeof($this->requests) < $this->window_size)
+            $this->window_size = sizeof($this->requests);
+
+        if ($this->window_size < 2) {
+            throw new RollingCurlException("Window size must be greater than 1");
+        }
+
+        $master = curl_multi_init();
+
+        // start the first batch of requests
+        for ($i = 0; $i < $this->window_size; $i++) {
+            $ch = curl_init();
+
+            $options = $this->get_options($this->requests[$i]);
+
+            curl_setopt_array($ch, $options);
+            curl_multi_add_handle($master, $ch);
+
+            // Add to our request Maps
+            $key = (string) $ch;
+            $this->requestMap[$key] = $i;
+        }
+
+        do {
+            while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM) ;
+            if ($execrun != CURLM_OK)
+                break;
+            // a request was just completed -- find out which one
+            while ($done = curl_multi_info_read($master)) {
+
+                // get the info and content returned on the request
+                $info = curl_getinfo($done['handle']);
+                $output = curl_multi_getcontent($done['handle']);
+
+                // send the return values to the callback function.
+                $callback = $this->callback;
+                if (is_callable($callback)) {
+                    $key = (string) $done['handle'];
+                    $request = $this->requests[$this->requestMap[$key]];
+                    unset($this->requestMap[$key]);
+                    call_user_func($callback, $output, $info, $request);
+                }
+
+                // start a new request (it's important to do this before removing the old one)
+                if ($i < sizeof($this->requests) && isset($this->requests[$i]) && $i < count($this->requests)) {
+                    $ch = curl_init();
+                    $options = $this->get_options($this->requests[$i]);
+                    curl_setopt_array($ch, $options);
+                    curl_multi_add_handle($master, $ch);
+
+                    // Add to our request Maps
+                    $key = (string) $ch;
+                    $this->requestMap[$key] = $i;
+                    $i++;
+                }
+
+                // remove the curl handle that just completed
+                curl_multi_remove_handle($master, $done['handle']);
+
+            }
+
+            // Block for data in / output; error handling is done by curl_multi_exec
+            if ($running)
+                curl_multi_select($master, $this->timeout);
+
+        } while ($running);
+        curl_multi_close($master);
+        return true;
+    }
+
+
+    /**
+     * Helper function to set up a new request by setting the appropriate options
+     *
+     * @access private
+     * @param Request $request
+     * @return array
+     */
+    private function get_options($request) {
+        // options for this entire curl object
+        $options = $this->__get('options');
+        if (ini_get('safe_mode') == 'Off' || !ini_get('safe_mode')) {
+            $options[CURLOPT_FOLLOWLOCATION] = 1;
+            $options[CURLOPT_MAXREDIRS] = 5;
+        }
+        $headers = $this->__get('headers');
+
+        // append custom options for this specific request
+        if ($request->options) {
+            $options = $request->options + $options;
+        }
+
+        // set the request URL
+        $options[CURLOPT_URL] = $request->url;
+
+        // posting data w/ this request?
+        if ($request->post_data) {
+            $options[CURLOPT_POST] = 1;
+            $options[CURLOPT_POSTFIELDS] = $request->post_data;
+        }
+        if ($headers) {
+            $options[CURLOPT_HEADER] = 0;
+            $options[CURLOPT_HTTPHEADER] = $headers;
+        }
+
+        return $options;
+    }
+
+    /**
+     * @return void
+     */
+    public function __destruct() {
+        unset($this->window_size, $this->callback, $this->options, $this->headers, $this->requests);
+    }
+}
+

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/RollingCurlGroup.php.svn-base
@@ -1,1 +1,218 @@
-
+group = $group;
+    }
+
+    /**
+     * Process the request
+     *
+     *
+     */
+    function process($output, $info) {
+        if ($this->group)
+            $this->group->process($output, $info, $this);
+    }
+
+    /**
+     * @return void
+     */
+    public function __destruct() {
+        unset($this->group);
+        parent::__destruct();
+    }
+
+}
+
+/**
+ * A group of curl requests.
+ *
+ * @throws RollingCurlGroupException *
+ */
+class RollingCurlGroup {
+    /**
+     * @var string group name
+     */
+    protected $name;
+
+    /**
+     * @var int total number of requests in a group
+     */
+    protected $num_requests = 0;
+
+    /**
+     * @var int total number of finished requests in a group
+     */
+    protected $finished_requests = 0;
+
+    /**
+     * @var array requests array
+     */
+    private $requests = array();
+
+    /**
+     * @param string $name group name
+     * @return void
+     */
+    function __construct($name) {
+        $this->name = $name;
+    }
+
+    /**
+     * @return void
+     */
+    public function __destruct() {
+        unset($this->name, $this->num_requests, $this->finished_requests, $this->requests);
+    }
+
+    /**
+     * Adds request to a group
+     *
+     * @throws RollingCurlGroupException
+     * @param RollingCurlGroupRequest|array $request
+     * @return bool
+     */
+    function add($request) {
+        if ($request instanceof RollingCurlGroupRequest) {
+            $request->setGroup($this);
+            $this->num_requests++;
+            $this->requests[] = $request;
+        }
+        else if (is_array($request)) {
+            foreach ($request as $req)
+            $this->add($req);
+        }
+        else
+            throw new RollingCurlGroupException("add: Request needs to be of instance RollingCurlGroupRequest");
+
+        return true;
+    }
+
+    /**
+     * @throws RollingCurlGroupException
+     * @param RollingCurl $rc
+     * @return bool
+     */
+    function addToRC(RollingCurl $rc){
+        $ret = true;
+
+        while (count($this->requests) > 0){
+            $ret1 = $rc->add(array_shift($this->requests));
+            if (!$ret1)
+                $ret = false;
+        }
+
+        return $ret;
+    }
+
+    /**
+     * Override to implement custom response processing.
+     *
+     * Don't forget to call parent::process().
+     *
+     * @param string $output received page body
+     * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc.
+     * @param RollingCurlRequest $request request used
+     * @return void
+     */
+    function process($output, $info, $request) {
+        $this->finished_requests++;
+
+        if ($this->finished_requests >= $this->num_requests)
+            $this->finished();
+    }
+
+    /**
+     * Override to execute code after all requests in a group are processed.
+     *
+     * @return void
+     */
+    function finished() {
+    }
+
+}
+
+/**
+ * Group version of rolling curl
+ */
+class GroupRollingCurl extends RollingCurl {
+
+    /**
+     * @var mixed common callback for all groups
+     */
+    private $group_callback = null;
+
+    /**
+     * @param string $output received page body
+     * @param array $info holds various information about response such as HTTP response code, content type, time taken to make request etc.
+     * @param RollingCurlRequest $request request used
+     * @return void
+     */
+    protected function process($output, $info, $request) {
+        if ($request instanceof RollingCurlGroupRequest)
+            $request->process($output, $info);
+
+        if (is_callable($this->group_callback))
+            call_user_func($this->group_callback, $output, $info, $request);
+    }
+
+    /**
+     * @param mixed $callback common callback for all groups
+     * @return void
+     */
+    function __construct($callback = null) {
+        $this->group_callback = $callback;
+
+        parent::__construct(array(&$this, "process"));
+    }
+
+    /**
+     * Adds a group to processing queue
+     *
+     * @param RollingCurlGroup|Request $request
+     * @return bool
+     */
+    public function add($request) {
+        if ($request instanceof RollingCurlGroup)
+            return $request->addToRC($this);
+        else
+            return parent::add($request);
+    }
+
+    /**
+     * Execute processing
+     *
+     * @param int $window_size Max number of simultaneous connections
+     * @return bool|string
+     */
+    public function execute($window_size = null) {
+        if (count($this->requests) == 0)
+            return false;
+
+        return parent::execute($window_size);
+    }
+}
+

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/example.php.svn-base
@@ -1,1 +1,66 @@
+(.*?)~i", $response, $out)) {
+		$title = $out[1];
+	}
+	echo "$title
";
+	print_r($info);
+    print_r($request);
+	echo "
";
+}
+
+// single curl request
+$rc = new RollingCurl("request_callback");
+$rc->request("http://www.msn.com");
+$rc->execute();
+
+// another single curl request
+$rc = new RollingCurl("request_callback");
+$rc->request("http://www.google.com");
+$rc->execute();
+
+echo "
";
+
+// top 20 sites according to alexa (11/5/09)
+$urls = array("http://www.google.com",
+              "http://www.facebook.com",
+              "http://www.yahoo.com",
+              "http://www.youtube.com",
+              "http://www.live.com",
+              "http://www.wikipedia.com",
+              "http://www.blogger.com",
+              "http://www.msn.com",
+              "http://www.baidu.com",
+              "http://www.yahoo.co.jp",
+              "http://www.myspace.com",
+              "http://www.qq.com",
+              "http://www.google.co.in",
+              "http://www.twitter.com",
+              "http://www.google.de",
+              "http://www.microsoft.com",
+              "http://www.google.cn",
+              "http://www.sina.com.cn",
+              "http://www.wordpress.com",
+              "http://www.google.co.uk");
+
+$rc = new RollingCurl("request_callback");
+$rc->window_size = 20;
+foreach ($urls as $url) {
+    $request = new RollingCurlRequest($url);
+    $rc->add($request);
+}
+$rc->execute();
+

--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/example_groups.php.svn-base
@@ -1,1 +1,49 @@
+url . "\n";
+        if ($this->test_verbose)
+            print_r($info);
+
+        parent::process($output, $info);
+    }
+}
+
+class TestCurlGroup extends RollingCurlGroup {
+    function process($output, $info, $request) {
+        echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n";
+        parent::process($output, $info, $request);
+    }
+
+    function finished() {
+        echo "Group CB: Finished" . $this->name . "\n";
+        parent::finished();
+    }
+}
+
+$group = new TestCurlGroup("High");
+$group->add(new TestCurlRequest("www.google.de"));
+$group->add(new TestCurlRequest("www.yahoo.de"));
+$group->add(new TestCurlRequest("www.newyorktimes.com"));
+$reqs[] = $group;
+
+$group = new TestCurlGroup("Normal");
+$group->add(new TestCurlRequest("twitter.com"));
+$group->add(new TestCurlRequest("www.bing.com"));
+$group->add(new TestCurlRequest("m.facebook.com"));
+$reqs[] = $group;
+
+$reqs[] = new TestCurlRequest("www.kernel.org");
+
+// No callback here, as its done in Request class
+$rc = new GroupRollingCurl();
+
+foreach ($reqs as $req)
+    $rc->add($req);
+