Parallelise http transfers for trip planner tester
--- /dev/null
+++ b/aws/awsStartup.sh
@@ -1,1 +1,37 @@
+#!/bin/bash
+#this script should be run from a fresh git checkout from github
+#ami base must have yum install lighttpd-fastcgi, git, tomcat6
+#screen php-cli php-gd tomcat6-webapps tomcat6-admin-webapps svn maven2
+#postgrtes postgres-server php-pg
+#http://www.how2forge.org/installing-lighttpd-with-php5-and-mysql-support-on-fedora-12
+cp /root/aws.php /tmp/
+mkdir /var/www/lib/staticmaplite/cache
+chcon -h system_u:object_r:httpd_sys_content_t /var/www
+chcon -R -h root:object_r:httpd_sys_content_t /var/www/*
+chcon -R -t httpd_sys_content_rw_t /var/www/lib/staticmaplite/cache
+chmod -R 777 /var/www/lib/staticmaplite/cache
+wget http://s3-ap-southeast-1.amazonaws.com/busresources/cbrfeed.zip \
+-O /var/www/cbrfeed.zip
+
+createdb transitdata
+createlang -d transitdata plpgsql
+psql -d transitdata -f /var/www/lib/postgis.sql
+# curl https://github.com/maxious/ACTBus-ui/raw/master/transitdata.cbrfeed.sql.gz -o transitdata.cbrfeed.sql.gz
+#made with pg_dump transitdata | gzip -c > transitdata.cbrfeed.sql.gz
+gunzip /var/www/transitdata.cbrfeed.sql.gz
+psql -d transitdata -f /var/www/transitdata.cbrfeed.sql
+#createuser transitdata -SDRP
+#password transitdata
+#psql -d transitdata -c \"GRANT SELECT ON TABLE agency,calendar,calendar_dates,routes,stop_times,stops,trips TO transitdata;\"
+php /var/www/updatedb.php
+
+wget http://s3-ap-southeast-1.amazonaws.com/busresources/Graph.obj \
+-O /tmp/Graph.obj
+rm -rfv /usr/share/tomcat6/webapps/opentripplanner*
+wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-webapp.war \
+-O /usr/share/tomcat6/webapps/opentripplanner-webapp.war
+wget http://s3-ap-southeast-1.amazonaws.com/busresources/opentripplanner-api-webapp.war \
+-O /usr/share/tomcat6/webapps/opentripplanner-api-webapp.war
+/etc/init.d/tomcat6 restart
+
--- /dev/null
+++ b/aws/pg_hba.conf
@@ -1,1 +1,75 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the
+# PostgreSQL documentation for a complete description
+# of this file. A short synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access. Records take one of these forms:
+#
+# local DATABASE USER METHOD [OPTIONS]
+# host DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
+# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
+# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain socket,
+# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an
+# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", a database name, or
+# a comma-separated list thereof.
+#
+# USER can be "all", a user name, a group name prefixed with "+", or
+# a comma-separated list thereof. In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names from
+# a separate file.
+#
+# CIDR-ADDRESS specifies the set of hosts the record matches.
+# It is made up of an IP address and a CIDR mask that is an integer
+# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies
+# the number of significant bits in the mask. Alternatively, you can write
+# an IP address and netmask in separate columns to specify the set of hosts.
+#
+# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", "krb5",
+# "ident", "pam", "ldap" or "cert". Note that "password" sends passwords
+# in clear text; "md5" is preferred since it sends encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE. The available options depend on the different authentication
+# methods - refer to the "Client Authentication" section in the documentation
+# for a list of which options are available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other special
+# characters must be quoted. Quoting one of the keywords "all", "sameuser" or
+# "samerole" makes the name lose its special character, and just match a
+# database or username with that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect. You can use
+# "pg_ctl reload" to do that.
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records. In that case you will also need to make PostgreSQL listen
+# on a non-local interface via the listen_addresses configuration parameter,
+# or via the -i or -h command line switches.
+#
+
+
+
+# TYPE DATABASE USER CIDR-ADDRESS METHOD
+
+# "local" is for Unix domain socket connections only
+local all all trust
+# IPv4 local connections:
+host all all 127.0.0.1/32 trust
+# IPv6 local connections:
+host all all ::1/128 trust
+
--- /dev/null
+++ b/include/common-db.inc.php
@@ -1,1 +1,21 @@
+<?php
+ if (php_uname('n') == "actbus-www") {
+ $conn = pg_connect("dbname=transitdata user=transitdata password=transitdata host=db.actbus.dotcloud.com port=2242");
+ } else if (isDebugServer()) {
+ $conn = pg_connect("dbname=transitdata user=postgres password=snmc");
+ } else {
+ $conn = pg_connect("dbname=transitdata user=transitdata password=transitdata ");
+ }
+ if (!$conn) {
+ die("A database error occurred.\n");
+ }
+
+ function databaseError($errMsg) {
+ die($errMsg);
+ }
+
+ include('db/route-dao.inc.php');
+ include('db/trip-dao.inc.php');
+ include('db/stop-dao.inc.php');
+ ?>
--- a/include/db/route-dao.inc.php
+++ b/include/db/route-dao.inc.php
@@ -40,6 +40,7 @@
}
function getRouteNextTrip($routeID) {
+ global $conn;
$query = "select * from routes join trips on trips.route_id = routes.route_id
join stop_times on stop_times.trip_id = trips.trip_id where
arrival_time > CURRENT_TIME and routes.route_id = '$routeID' order by
@@ -52,7 +53,19 @@
}
return pg_fetch_assoc($result);
}
-
+function getRouteTrips($routeID) {
+ global $conn;
+ $query = "select * from routes join trips on trips.route_id = routes.route_id
+join stop_times on stop_times.trip_id = trips.trip_id where routes.route_id = '$routeID' order by
+arrival_time ";
+ debug($query,"database");
+ $result = pg_query($conn, $query);
+ if (!$result) {
+ databaseError(pg_result_error($result));
+ return Array();
+ }
+ return pg_fetch_all($result);
+ }
function getRoutesByDestination($destination = "", $service_period = "") {
global $conn;
if ($service_period == "") $service_period = service_period();
--- a/include/db/trip-dao.inc.php
+++ b/include/db/trip-dao.inc.php
@@ -2,7 +2,10 @@
function getTrip($tripID)
{
global $conn;
- $query = "Select * from trips where trip_id = '$tripID' join routes on trips.route_id = routes.route_id LIMIT 1";
+ $query = "Select * from trips
+ join routes on trips.route_id = routes.route_id
+ where trip_id = '$tripID'
+ LIMIT 1";
debug($query, "database");
$result = pg_query($conn, $query);
if (!$result) {
@@ -49,7 +52,7 @@
function getTimeInterpolatedTrip($tripID)
{
global $conn;
- $query = "SELECT stop_times.trip_id,arrival_time,stop_times.stop_id,stop_lat,stop_lon,
+ $query = "SELECT stop_times.trip_id,arrival_time,stop_times.stop_id,stop_lat,stop_lon,stop_name,stop_code,
stop_sequence,service_id,trips.route_id,route_short_name,route_long_name
FROM stop_times
join trips on trips.trip_id = stop_times.trip_id
@@ -129,14 +132,14 @@
$r = pg_fetch_assoc($result);
return $r['arrival_time'];
}
-function viaPointNames($tripid, $stopid)
+function viaPointNames($tripid, $stop_sequence = "")
{
global $conn;
$query = "SELECT stop_name
FROM stop_times join stops on stops.stop_id = stop_times.stop_id
WHERE stop_times.trip_id = '$tripid'
-AND stop_sequence > '$stop_sequence'
-AND substr(stop_code,1,2) != 'Wj' ORDER BY stop_sequence";
+".($stop_sequence != "" ? "AND stop_sequence > '$stop_sequence'" : "").
+"AND substr(stop_code,1,2) != 'Wj' ORDER BY stop_sequence";
debug($query, "database");
$result = pg_query($conn, $query);
if (!$result) {
--- a/labs/tripPlannerTester.kml.php
+++ b/labs/tripPlannerTester.kml.php
@@ -8,6 +8,50 @@
}
else {
return (($pBegin - $pEnd) * (1 - ($pStep / $pMax))) + $pEnd;
+ }
+}
+require ("../lib/rolling-curl/RollingCurl.php");
+function processResult_cb($response, $info, $request)
+{
+ global $testRegions, $regionTimes,$csv,$kml, $latdeltasize,$londeltasize;
+ $md = $request->metadata;
+ $tripplan = json_decode($response);
+ $plans = Array();
+ //var_dump(Array($info, $request));
+ if (is_array($tripplan->plan->itineraries->itinerary)) {
+ foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) {
+ $plans[floor($itinerary->duration / 60000) ] = $itinerary;
+ }
+ }
+ else {
+ $plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary;
+ }
+ if ($csv) echo "{$md['i']},{$md['j']}," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,{$md['key']}\n";
+ if ($kml) {
+ $time = min(array_keys($plans));
+ $plan = "";
+ if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) {
+ foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) {
+ $plan.= processLeg($legNumber, $leg) . ",";
+ }
+ }
+ else {
+ $plan.= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg);
+ }
+ if (isset($tripplan->error) && $tripplan->error->id == 404) {
+ $time = 999;
+ $plan = "Trip not possible without excessive walking from nearest bus stop";
+ }
+ $testRegions[] = Array(
+ "lat" => $md['i'],
+ "lon" => $md['j'],
+ "time" => $time,
+ "latdeltasize" => $latdeltasize,
+ "londeltasize" => $londeltasize,
+ "regionname" => $md['key'],
+ "plan" => $plan . "<br/><a href='" . htmlspecialchars($url) . "'>original plan</a>"
+ );
+ $regionTimes[] = $time;
}
}
function Gradient($HexFrom, $HexTo, $ColorSteps)
@@ -49,18 +93,17 @@
//}
//$walkingstep.= floor($step->distance) . "m";
//return $walkingstep;
+
}
}
$csv = false;
$kml = true;
if ($kml) {
- //header('Content-Type: application/vnd.google-earth.kml+xml');
+ header('Content-Type: application/vnd.google-earth.kml+xml');
echo '<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2"><Document>';
}
include ('../include/common.inc.php');
-//Test code to grab transit times
-// make sure to sleep(10);
$boundingBoxes = Array(
"belconnen" => Array(
"startlat" => - 35.1928,
@@ -105,72 +148,29 @@
$useragent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1";
if ($csv) echo "<pre>";
if ($csv) echo "lat,lon,time,latdeltasize, londeltasize, region key name\n";
+$rc = new RollingCurl("processResult_cb");
+$rc->window_size = 3;
foreach ($boundingBoxes as $key => $boundingBox) {
for ($i = $boundingBox['startlat']; $i >= $boundingBox['finishlat']; $i-= $latdeltasize) {
for ($j = $boundingBox['startlon']; $j <= $boundingBox['finishlon']; $j+= $londeltasize) {
$url = $otpAPIurl . "ws/plan?date=" . urlencode($startDate) . "&time=" . urlencode($startTime) . "&mode=TRANSIT%2CWALK&optimize=QUICK&maxWalkDistance=440&wheelchair=false&toPlace=" . $i . "," . $j . "&fromPlace=$fromPlace";
- $ch = curl_init($url);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
- curl_setopt($ch, CURLOPT_HEADER, 0);
- curl_setopt($ch, CURLOPT_HTTPHEADER, array(
+ $request = new RollingCurlRequest($url);
+ $request->headers = Array(
"Accept: application/json"
- ));
- curl_setopt($ch, CURLOPT_TIMEOUT, 5);
- $page = curl_exec($ch);
- if (curl_errno($ch)) {
- if ($csv) echo "Trip planner temporarily unavailable: " . curl_errno($ch) . " " . curl_error($ch);
- }
- else {
- $tripplan = json_decode($page);
- $plans = Array();
- if (is_array($tripplan->plan->itineraries->itinerary)) {
- foreach ($tripplan->plan->itineraries->itinerary as $itineraryNumber => $itinerary) {
- $plans[floor($itinerary->duration / 60000) ] = $itinerary;
- }
- }
- else {
- $plans[floor($tripplan->plan->itineraries->itinerary->duration / 60000) ] = $tripplan->plan->itineraries->itinerary;
- }
- if ($csv) echo "$i,$j," . min(array_keys($plans)) . ",$latdeltasize, $londeltasize,$key\n";
- if ($kml) {
- $time = min(array_keys($plans));
- $plan = "";
- if (is_array($plans[min(array_keys($plans)) ]->legs->leg)) {
- foreach ($plans[min(array_keys($plans)) ]->legs->leg as $legNumber => $leg) {
- $plan .= processLeg($legNumber, $leg).",";
- }
- }
- else {
- $plan .= processLeg(0, $plans[min(array_keys($plans)) ]->legs->leg);
- }
- if (isset($tripplan->error) && $tripplan->error->id == 404) {
- $time = 999;
- $plan = "Trip not possible without excessive walking from nearest bus stop";
- }
- $testRegions[] = Array(
- "lat" => $i,
- "lon" => $j,
- "time" => $time,
- "latdeltasize" => $latdeltasize,
- "londeltasize" => $londeltasize,
- "regionname" => $key,
- "plan" => $plan . "<br/><a href='". htmlspecialchars($url)."'>original plan</a>"
- );
- $regionTimes[] = $time;
- }
- }
- flush(); @ob_flush();
- curl_close($ch);
- }
- }
-}
+ );
+ $request->metadata = Array( "i" => $i, "j" => $j, "key" => $key);
+ $rc->add($request);
+ }
+ }
+}
+$rc->execute();
if ($kml) {
$colorSteps = 9;
//$minTime = min($regionTimes);
//$maxTime = max($regionTimes);
//$rangeTime = $maxTime - $minTime;
//$deltaTime = $rangeTime / $colorSteps;
- $Gradients = Gradient(strrev("66FF00"), strrev("FF0000"), $colorSteps); // KML is BGR not RGB so strrev
+ $Gradients = Gradient(strrev("66FF00") , strrev("FF0000") , $colorSteps); // KML is BGR not RGB so strrev
foreach ($testRegions as $testRegion) {
//$band = (floor(($testRegion[time] - $minTime) / $deltaTime));
$band = (floor($testRegion[time] / 10));
--- /dev/null
+++ b/lib/rolling-curl/.svn/all-wcprops
@@ -1,1 +1,42 @@
+K 25
+svn:wc:ra_dav:version-url
+V 22
+/svn/!svn/ver/20/trunk
+END
+RollingCurlGroup.php
+K 25
+svn:wc:ra_dav:version-url
+V 43
+/svn/!svn/ver/20/trunk/RollingCurlGroup.php
+END
+example_groups.php
+K 25
+svn:wc:ra_dav:version-url
+V 41
+/svn/!svn/ver/20/trunk/example_groups.php
+END
+example.php
+K 25
+svn:wc:ra_dav:version-url
+V 34
+/svn/!svn/ver/20/trunk/example.php
+END
+RollingCurl.php
+K 25
+svn:wc:ra_dav:version-url
+V 38
+/svn/!svn/ver/20/trunk/RollingCurl.php
+END
+CHANGELOG.txt
+K 25
+svn:wc:ra_dav:version-url
+V 36
+/svn/!svn/ver/20/trunk/CHANGELOG.txt
+END
+README.txt
+K 25
+svn:wc:ra_dav:version-url
+V 33
+/svn/!svn/ver/20/trunk/README.txt
+END
--- /dev/null
+++ b/lib/rolling-curl/.svn/entries
@@ -1,1 +1,233 @@
-
+10
+
+dir
+20
+http://rolling-curl.googlecode.com/svn/trunk
+http://rolling-curl.googlecode.com/svn
+
+
+
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+74aa2acc-2e27-11de-b2a4-4f96ceaaac44
+
+RollingCurlGroup.php
+file
+
+
+
+
+2011-04-10T08:32:48.081650Z
+73c08d9e9e24b4adc89816624c7aca30
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+5152
+
+example_groups.php
+file
+
+
+
+
+2011-04-10T08:32:48.082650Z
+907ed82a47d346c39acbd5578e1d0230
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1367
+
+example.php
+file
+
+
+
+
+2011-04-10T08:32:48.083650Z
+87aa845abfaffc09ed4eca024f2a8b8a
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+1860
+
+RollingCurl.php
+file
+
+
+
+
+2011-04-10T08:32:48.084650Z
+205391c449f3f3ee050004dadc374dc8
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+10444
+
+CHANGELOG.txt
+file
+
+
+
+
+2011-04-10T08:32:48.085650Z
+d0452f6f9530ed04580159121d0fd5f7
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+has-props
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+662
+
+README.txt
+file
+
+
+
+
+2011-04-10T08:32:48.085650Z
+60dd357081431c0f2b82989cdbce8615
+2010-09-12T20:39:22.711474Z
+20
+alexander.makarow
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+6355
+
+
--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/CHANGELOG.txt.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/RollingCurlGroup.php.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
--- /dev/null
+++ b/lib/rolling-curl/.svn/prop-base/example_groups.php.svn-base
@@ -1,1 +1,6 @@
+K 13
+svn:eol-style
+V 6
+native
+END
--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/CHANGELOG.txt.svn-base
@@ -1,1 +1,15 @@
+Rolling Curl changelog
+======================
+September 13, 2010
+------------------
+- Bug #12, #14: Fixed default options overriding (LionsAd)
+- Bug #10: Added use of curl_multi_select to avoid burning CPU (LionsAd)
+- Enh #6, #9: Added $request as parameter to callback function (LionsAd)
+- Chg: Request renamed to RollingCurlRequest (LionsAd)
+- Added RollingCurlGroup class that allows processing groups of requests (LionsAd)
+- More cleanup at unsetting a class (LionsAd)
+- Timeout parameter for curl_multi_select is now configurable (LionsAd)
+- single_curl now returns true (LionsAd)
+- Readme corrections (Alexander Makarov)
+- Code cleanup (Alexander Makarov)
--- /dev/null
+++ b/lib/rolling-curl/.svn/text-base/README.txt.svn-base
@@ -1,1 +1,210 @@
-
+Rolling Curl
+============
+
+RollingCurl allows you to process multiple HTTP requests in parallel using CURL PHP library.
+
+Released under the Apache License 2.0.
+
+Authors
+-------
+- Was originally written by [Josh Fraser](joshfraser.com).
+- Currently maintained by [Alexander Makarov](http://rmcreative.ru/).
+- Received significant updates and patched from [LionsAd](http://github.com/LionsAd/rolling-curl).
+
+Overview
+--------
+RollingCurl is a more efficient implementation of curl_multi() curl_multi is a great way to process multiple HTTP requests in parallel in PHP.
+curl_multi is particularly handy when working with large data sets (like fetching thousands of RSS feeds at one time). Unfortunately there is
+very little documentation on the best way to implement curl_multi. As a result, most of the examples around the web are either inefficient or
+fail entirely when asked to handle more than a few hundred requests.
+
+The problem is that most implementations of curl_multi wait for each set of requests to complete before processing them. If there are too many requests
+to process at once, they usually get broken into groups that are then processed one at a time. The problem with this is that each group has to wait for
+the slowest request to download. In a group of 100 requests, all it takes is one slow one to delay the processing of 99 others. The larger the number of
+requests you are dealing with, the more noticeable this latency becomes.
+
+The solution is to process each request as soon as it completes. This eliminates the wasted CPU cycles from busy waiting. Also there is a queue of
+cURL requests to allow for maximum throughput. Each time a request is completed, a new one is added from the queue. By dynamically adding and removing
+links, we keep a constant number of links downloading at all times. This gives us a way to throttle the amount of simultaneous requests we are sending.
+The result is a faster and more efficient way of processing large quantities of cURL requests in parallel.
+
+Callbacks
+---------
+
+Each of requests usually do have a callback to process results that is being executed when request is done
+(both successfully or not).
+
+Callback accepts three parameters and can look like the following one:
+~~~
+[php]
+function request_callback($response, $info, $request){
+ // doing something with the data received
+}
+~~~
+
+- $response contains received page body.
+- $info is an associative array that holds various information about response such as HTTP response code, content type,
+time taken to make request etc.
+- $request contains RollingCurlRequest that was used to make request.
+
+Examples
+--------
+### Hello world
+
+~~~
+[php]
+// an array of URL's to fetch
+$urls = array("http://www.google.com",
+ "http://www.facebook.com",
+ "http://www.yahoo.com");
+
+// a function that will process the returned responses
+function request_callback($response, $info, $request) {
+ // parse the page title out of the returned HTML
+ if (preg_match("~<title>(.*?)</title>~i", $response, $out)) {
+ $title = $out[1];
+ }
+ echo "<b>$title</b><br />";
+ print_r($info);
+ echo "<hr>";
+}
+
+// create a new RollingCurl object and pass it the name of your custom callback function
+$rc = new RollingCurl("request_callback");
+// the window size determines how many simultaneous requests to allow.
+$rc->window_size = 20;
+foreach ($urls as $url) {
+ // add each request to the RollingCurl object
+ $request = new RollingCurlRequest($url);
+ $rc->add($request);
+}
+$rc->execute();
+~~~
+
+
+### Setting custom options
+
+Set custom options for EVERY request:
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$rc->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
+$rc->execute();
+~~~
+
+Set custom options for A SINGLE request:
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$request = new RollingCurlRequest($url);
+$request->options = array(CURLOPT_HEADER => true, CURLOPT_NOBODY => true);
+$rc->add($request);
+$rc->execute();
+~~~
+
+### Shortcuts
+
+~~~
+[php]
+$rc = new RollingCurl("request_callback");
+$rc->get("http://www.google.com");
+$rc->get("http://www.yahoo.com");
+$rc->execute();
+~~~
+
+### Class callbacks
+
+~~~
+[php]
+class MyInfoCollector {
+ private $rc;
+
+ function __construct(){
+ $this->rc = new RollingCurl(array($this, 'processPage'));
+ }
+
+ function processPage($response, $info, $request){
+ //...
+ }
+
+ function run($urls){
+ foreach ($urls as $url){
+ $request = new RollingCurlRequest($url);
+ $this->rc->add($request);
+ }
+ $this->rc->execute();
+ }
+}
+
+$collector = new MyInfoCollector();
+$collector->run(array(
+ 'http://google.com/',
+ 'http://yahoo.com/'
+));
+~~~
+
+### Using RollingCurlGroup
+
+~~~
+[php]
+class TestCurlRequest extends RollingCurlGroupRequest {
+ public $test_verbose = true;
+
+ function process($output, $info) {
+ echo "Processing " . $this->url . "\n";
+ if ($this->test_verbose)
+ print_r($info);
+
+ parent::process($output, $info);
+ }
+}
+
+class TestCurlGroup extends RollingCurlGroup {
+ function process($output, $info, $request) {
+ echo "Group CB: Progress " . $this->name . " (" . ($this->finished_requests + 1) . "/" . $this->num_requests . ")\n";
+ parent::process($output, $info, $request);
+ }
+
+ function finished() {
+ echo "Group CB: Finished" . $this->name . "\n";
+ parent::finished();
+ }
+}
+
+$group = new TestCurlGroup("High");
+$group->add(new TestCurlRequest("www.google.de"));
+$group->add(new TestCurlRequest("www.yahoo.de"));
+$group->add(new TestCurlRequest("www.newyorktimes.com"));
+$reqs[] = $group;
+
+$group = new TestCurlGroup("Normal");
+$group->add(new TestCurlRequest("twitter.com"));
+$group->add(new TestCurlRequest("www.bing.com"));
+$group->add(new TestCurlRequest("m.facebook.com"));
+$reqs[] = $group;
+
+$reqs[] = new TestCurlRequest("www.kernel.org");
+
+// No callback here, as its done in Request class
+$rc = new GroupRollingCurl();
+
+foreach ($reqs as $req)
+$rc->add($req);
+
+$rc->execute();
+~~~
+
+The same function (add) can be used both for adding requests and groups of requests.
+The "callback" in request and groups is:
+
+process($output, $info)
+
+and
+
+process($output, $info, $request)
+
+Also you can override RollingCu