projects / busui.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Beginning of twitter interaction in service alerts component
Beginning of twitter interaction in service alerts component

file:a/aws/busuidb.sh -> file:b/aws/busuidb.sh
createdb transitdata createdb transitdata
createlang -d transitdata plpgsql createlang -d transitdata plpgsql
psql -d transitdata -f /var/www/lib/postgis.sql psql -d transitdata -f /var/www/lib/postgis.sql
# curl https://github.com/maxious/ACTBus-ui/raw/master/transitdata.cbrfeed.sql.gz -o transitdata.cbrfeed.sql.gz # curl https://github.com/maxious/ACTBus-ui/raw/master/transitdata.cbrfeed.sql.gz -o transitdata.cbrfeed.sql.gz
#made with pg_dump transitdata | gzip -c > transitdata.cbrfeed.sql.gz #made with pg_dump transitdata | gzip -c > transitdata.cbrfeed.sql.gz
gunzip /var/www/transitdata.cbrfeed.sql.gz gunzip /var/www/transitdata.cbrfeed.sql.gz
psql -d transitdata -f /var/www/transitdata.cbrfeed.sql psql -d transitdata -f /var/www/transitdata.cbrfeed.sql
#createuser transitdata -SDRP #createuser transitdata -SDRP
#password transitdata #password transitdata
#psql -d transitdata -c "GRANT SELECT ON TABLE agency,calendar,calendar_dates,routes,stop_times,stops,trips TO transitdata;" #psql -d transitdata -c "GRANT SELECT ON TABLE agency,calendar,calendar_dates,routes,stop_times,stops,trips TO transitdata;"
#psql -d transitdata -c "GRANT SELECT,INSERT ON TABLE myway_observations,myway_routes,myway_stops,myway_timingdeltas TO transitdata;" #psql -d transitdata -c "GRANT SELECT,INSERT ON TABLE myway_observations,myway_routes,myway_stops,myway_timingdeltas TO transitdata;"
#psql -d transitdata -c "GRANT SELECT,INSERT,UPDATE ON TABLE myway_routes,myway_stops TO transitdata;" #psql -d transitdata -c "GRANT SELECT,INSERT,UPDATE ON TABLE myway_routes,myway_stops TO transitdata;"
##psql -d transitdata -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO transitdata;" ##psql -d transitdata -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO transitdata;"
## INSERT INTO geometry_columns(f_table_catalog, f_table_schema, f_table_name, f_geometry_column, coord_dimension, srid, "type")  
##SELECT '', 'public', 'shapes', 'shape_pt', ST_CoordDim(shape_pt), ST_SRID(shape_pt), GeometryType(shape_pt)  
##FROM shapes LIMIT 1;  
php /var/www/updatedb.php php /var/www/updatedb.php
file:a/include/db/trip-dao.inc.php -> file:b/include/db/trip-dao.inc.php
<?php <?php
   
/* /*
* Copyright 2010,2011 Alexander Sadleir * Copyright 2010,2011 Alexander Sadleir
   
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at
   
http://www.apache.org/licenses/LICENSE-2.0 http://www.apache.org/licenses/LICENSE-2.0
   
Unless required by applicable law or agreed to in writing, software Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
   
function getTrip($tripID) { function getTrip($tripID) {
global $conn; global $conn;
$query = "Select * from trips $query = "Select * from trips
join routes on trips.route_id = routes.route_id join routes on trips.route_id = routes.route_id
where trip_id = :tripID where trip_id = :tripID
LIMIT 1"; LIMIT 1";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
   
return Array(); return Array();
} }
return $query->fetch(PDO :: FETCH_ASSOC); return $query->fetch(PDO :: FETCH_ASSOC);
} }
function getTripStops($tripID) {  
  function getTripShape($tripID) {
  // todo, use shapes table if shape_id specified
global $conn; global $conn;
$query = "SELECT stop_id, stop_name, ST_AsKML(position) as positionkml, $query = "SELECT ST_AsKML(ST_MakeLine(geometry(a.position))) as the_route
  FROM (SELECT position,
stop_sequence, trips.trip_id stop_sequence, trips.trip_id
FROM stop_times FROM stop_times
join trips on trips.trip_id = stop_times.trip_id join trips on trips.trip_id = stop_times.trip_id
join stops on stops.stop_id = stop_times.stop_id join stops on stops.stop_id = stop_times.stop_id
WHERE trips.trip_id = :tripID ORDER BY stop_sequence"; WHERE trips.trip_id = :tripID ORDER BY stop_sequence) as a group by a.trip_id";
debug($query, "database");  
$query = $conn->prepare($query);  
$query->bindParam(":tripID", $tripID);  
$query->execute();  
if (!$query) {  
databaseError($conn->errorInfo());  
return Array();  
}  
return $query->fetchColumn(0);  
}  
function getTripShape($tripID) {  
// todo, use shapes table if shape_id specified  
global $conn;  
$query = "SELECT ST_AsKML(ST_MakeLine(geometry(a.shape_pt))) as the_route  
FROM (SELECT shapes.shape_id,shape_pt from shapes  
inner join trips on shapes.shape_id = trips.shape_id  
WHERE trips.trip_id = :tripID ORDER BY shape_pt_sequence) as a group by a.shape_id";  
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
return $query->fetchColumn(0); return $query->fetchColumn(0);
} }
   
function getTripStopTimes($tripID) { function getTripStopTimes($tripID) {
global $conn; global $conn;
$query = "SELECT stop_times.trip_id,trip_headsign,arrival_time,stop_times.stop_id $query = "SELECT stop_times.trip_id,trip_headsign,arrival_time,stop_times.stop_id
,stop_lat,stop_lon,stop_name,stop_desc,stop_code, ,stop_lat,stop_lon,stop_name,stop_desc,stop_code,
stop_sequence,service_id,trips.route_id,route_short_name,route_long_name stop_sequence,service_id,trips.route_id,route_short_name,route_long_name
FROM stop_times FROM stop_times
join trips on trips.trip_id = stop_times.trip_id join trips on trips.trip_id = stop_times.trip_id
join routes on trips.route_id = routes.route_id join routes on trips.route_id = routes.route_id
join stops on stops.stop_id = stop_times.stop_id join stops on stops.stop_id = stop_times.stop_id
WHERE trips.trip_id = :tripID $range ORDER BY stop_sequence"; WHERE trips.trip_id = :tripID $range ORDER BY stop_sequence";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
$stopTimes = $query->fetchAll(); $stopTimes = $query->fetchAll();
return $stopTimes; return $stopTimes;
} }
   
function getTripAtStop($tripID, $stop_sequence) { function getTripAtStop($tripID, $stop_sequence) {
global $conn; global $conn;
foreach (getTripStopTimes($tripID) as $tripStop) { foreach (getTripStopTimes($tripID) as $tripStop) {
if ($tripStop['stop_sequence'] == $stop_sequence) if ($tripStop['stop_sequence'] == $stop_sequence)
return $tripStop; return $tripStop;
} }
return Array(); return Array();
} }
   
function getTripStartTime($tripID) { function getTripStartTime($tripID) {
global $conn; global $conn;
$query = "Select * from stop_times $query = "Select * from stop_times
where trip_id = :tripID where trip_id = :tripID
AND arrival_time IS NOT NULL AND arrival_time IS NOT NULL
AND stop_sequence = '1'"; AND stop_sequence = '1'";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
$r = $query->fetch(PDO :: FETCH_ASSOC); $r = $query->fetch(PDO :: FETCH_ASSOC);
return $r['arrival_time']; return $r['arrival_time'];
} }
   
function getTripEndTime($tripID) { function getTripEndTime($tripID) {
global $conn; global $conn;
$query = "SELECT trip_id,max(arrival_time) as arrival_time from stop_times $query = "SELECT trip_id,max(arrival_time) as arrival_time from stop_times
WHERE stop_times.arrival_time IS NOT NULL and trip_id = :tripID group by trip_id"; WHERE stop_times.arrival_time IS NOT NULL and trip_id = :tripID group by trip_id";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
$r = $query->fetch(PDO :: FETCH_ASSOC); $r = $query->fetch(PDO :: FETCH_ASSOC);
return $r['arrival_time']; return $r['arrival_time'];
} }
   
function getTripDestination($tripID) { function getTripDestination($tripID) {
global $conn; global $conn;
$query = "SELECT stops.stop_id, stops.stop_name, stops.stop_desc $query = "SELECT stops.stop_id, stops.stop_name, stops.stop_desc
from stop_times inner join stops on stop_times.stop_id = stops.stop_id from stop_times inner join stops on stop_times.stop_id = stops.stop_id
WHERE trip_id = :tripID order by stop_sequence desc limit 1"; WHERE trip_id = :tripID order by stop_sequence desc limit 1";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
$r = $query->fetch(PDO :: FETCH_ASSOC); $r = $query->fetch(PDO :: FETCH_ASSOC);
return $r; return $r;
} }
   
function getActiveTrips($time) { function getActiveTrips($time) {
global $conn; global $conn;
if ($time == "") if ($time == "")
$time = current_time(); $time = current_time();
$query = "Select distinct stop_times.trip_id, start_times.arrival_time as start_time, end_times.arrival_time as end_time from stop_times, (SELECT trip_id,arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL $query = "Select distinct stop_times.trip_id, start_times.arrival_time as start_time, end_times.arrival_time as end_time from stop_times, (SELECT trip_id,arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL
AND stop_sequence = '1') as start_times, (SELECT trip_id,max(arrival_time) as arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL group by trip_id) as end_times AND stop_sequence = '1') as start_times, (SELECT trip_id,max(arrival_time) as arrival_time from stop_times WHERE stop_times.arrival_time IS NOT NULL group by trip_id) as end_times
WHERE start_times.trip_id = end_times.trip_id AND stop_times.trip_id = end_times.trip_id AND :time > start_times.arrival_time AND :time < end_times.arrival_time"; WHERE start_times.trip_id = end_times.trip_id AND stop_times.trip_id = end_times.trip_id AND :time > start_times.arrival_time AND :time < end_times.arrival_time";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
$query->bindParam(":time", $time); $query->bindParam(":time", $time);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
return $query->fetchAll(); return $query->fetchAll();
} }
   
function viaPoints($tripID, $stop_sequence = "") { function viaPoints($tripID, $stop_sequence = "") {
global $conn; global $conn;
$query = "SELECT stops.stop_id, stop_name, arrival_time $query = "SELECT stops.stop_id, stop_name, arrival_time
FROM stop_times join stops on stops.stop_id = stop_times.stop_id FROM stop_times join stops on stops.stop_id = stop_times.stop_id
WHERE stop_times.trip_id = :tripID WHERE stop_times.trip_id = :tripID
" . ($stop_sequence != "" ? " AND stop_sequence > :stop_sequence " : "") . " ORDER BY stop_sequence"; " . ($stop_sequence != "" ? " AND stop_sequence > :stop_sequence " : "") . " ORDER BY stop_sequence";
debug($query, "database"); debug($query, "database");
$query = $conn->prepare($query); $query = $conn->prepare($query);
if ($stop_sequence != "") if ($stop_sequence != "")
$query->bindParam(":stop_sequence", $stop_sequence); $query->bindParam(":stop_sequence", $stop_sequence);
$query->bindParam(":tripID", $tripID); $query->bindParam(":tripID", $tripID);
$query->execute(); $query->execute();
if (!$query) { if (!$query) {
databaseError($conn->errorInfo()); databaseError($conn->errorInfo());
return Array(); return Array();
} }
return $query->fetchAll(); return $query->fetchAll();
} }
   
   
?> ?>
file:a/servicealerts/importer.py -> file:b/servicealerts/importer.py
#dependencies http://code.google.com/p/python-twitter/  
   
# info # info
# http://stackoverflow.com/questions/4206882/named-entity-recognition-with-preset-list-of-names-for-python-php/4207128#4207128 # http://stackoverflow.com/questions/4206882/named-entity-recognition-with-preset-list-of-names-for-python-php/4207128#4207128
# http://alias-i.com/lingpipe/demos/tutorial/ne/read-me.html approximate dist # http://alias-i.com/lingpipe/demos/tutorial/ne/read-me.html approximate dist
# http://streamhacker.com/2008/12/29/how-to-train-a-nltk-chunker/ more training # http://streamhacker.com/2008/12/29/how-to-train-a-nltk-chunker/ more training
# http://www.postgresql.org/docs/9.1/static/pgtrgm.html # http://www.postgresql.org/docs/9.1/static/pgtrgm.html
   
# data sources # data sources
# http://twitter.com/#!/ACTEmergencyInf instant site wide # http://twitter.com/#!/ACTEmergencyInf instant site wide
# http://twitter.com/#!/ACTPol_Traffic # http://twitter.com/#!/ACTPol_Traffic
# http://esa.act.gov.au/feeds/currentincidents.xml # http://esa.act.gov.au/feeds/currentincidents.xml
   
# source: https://gist.github.com/322906/90dea659c04570757cccf0ce1e6d26c9d06f9283 # source: https://gist.github.com/322906/90dea659c04570757cccf0ce1e6d26c9d06f9283
import nltk import nltk
import twitter import twitter
  import tweepy
import psycopg2 import psycopg2
  from iniparse import INIConfig
   
def insert_service_alert_sitewide(heading, message, url): def insert_service_alert_sitewide(heading, message, url):
  print "NaN"
   
def insert_service_alert_for_street(streets, heading, message, url): def insert_service_alert_for_street(streets, heading, message, url):
conn_string = "host='localhost' dbname='energymapper' user='postgres' password='snmc'" conn_string = "host='localhost' dbname='energymapper' user='postgres' password='snmc'"
# print the connection string we will use to connect # print the connection string we will use to connect
print "Connecting to database\n ->%s" % (conn_string) print "Connecting to database\n ->%s" % (conn_string)
try: try:
# get a connection, if a connect cannot be made an exception will be raised here # get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string) conn = psycopg2.connect(conn_string)
   
# conn.cursor will return a cursor object, you can use this cursor to perform queries # conn.cursor will return a cursor object, you can use this cursor to perform queries
cursor = conn.cursor() cursor = conn.cursor()
   
# execute our Query # execute our Query
cursor.execute("select max(value), extract(dow from max(time)) as dow, \ cursor.execute("")
extract(year from max(time))::text || lpad(extract(month from max(time))::text,2,'0') \  
|| lpad(extract(month from max(time))::text,2,'0') as yearmonthweek, to_char(max(time),'J') \  
from environmentdata_values where \"dataSourceID\"='NSWAEMODemand' \  
group by extract(dow from time), extract(year from time), extract(week from time) \  
order by extract(year from time), extract(week from time), extract(dow from time)")  
   
# retrieve the records from the database # retrieve the records from the database
records = cursor.fetchall() records = cursor.fetchall()
   
for record in records: for record in records:
ys.append(record[0]) ys.append(record[0])
# >>> cur.execute("INSERT INTO test (num, data) VALUES (%s, %s)", (42, 'bar')) # >>> cur.execute("INSERT INTO test (num, data) VALUES (%s, %s)", (42, 'bar'))
#>>> cur.statusmessage #>>> cur.statusmessage
#'INSERT 0 1' #'INSERT 0 1'
except: except:
# Get the most recent exception # Get the most recent exception
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
# Exit the script and print an error telling what happened. # Exit the script and print an error telling what happened.
sys.exit("Database connection failed!\n ->%s" % (exceptionValue)) sys.exit("Database connection failed!\n ->%s" % (exceptionValue))
def get_tweets(user):  
tapi = twitter.Api()  
return tapi.GetUserTimeline(user)  
   
def extract_entity_names(t): def extract_entity_names(t):
entity_names = [] entity_names = []
if hasattr(t, 'node') and t.node: if hasattr(t, 'node') and t.node:
if t.node == 'NE': if t.node == 'NE':
entity_names.append(' '.join([child[0] for child in t])) entity_names.append(' '.join([child[0] for child in t]))
else: else:
for child in t: for child in t:
entity_names.extend(extract_entity_names(child)) entity_names.extend(extract_entity_names(child))
return entity_names return entity_names
   
def extract_names(sample): def extract_names(sample):
sentences = nltk.sent_tokenize(sample) sentences = nltk.sent_tokenize(sample)
tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences] tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences] tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
chunked_sentences = nltk.batch_ne_chunk(tagged_sentences, binary=True) chunked_sentences = nltk.batch_ne_chunk(tagged_sentences, binary=True)
# chunked/tagged may be enough to just find and match the nouns # chunked/tagged may be enough to just find and match the nouns
   
entity_names = [] entity_names = []
for tree in chunked_sentences: for tree in chunked_sentences:
# Print results per sentence # Print results per sentence
# print extract_entity_names(tree) # print extract_entity_names(tree)
entity_names.extend(extract_entity_names(tree)) entity_names.extend(extract_entity_names(tree))
   
# Print all entity names # Print all entity names
#print entity_names #print entity_names
   
# Print unique entity names # Print unique entity names
print set(entity_names) print set(entity_names)
  cfg = INIConfig(open('/tmp/aws.ini'))
   
  auth = tweepy.OAuthHandler(cfg.api_keys.twitter_consumer_key, cfg.api_keys.twitter_consumer_secret)
  auth.set_access_token(cfg.api_keys.twitter_access_token, cfg.api_keys.twitter_access_token_secret)
   
  api = tweepy.API(auth)
   
  # If the authentication was successful, you should
  # see the name of the account print out
  print api.me().name
  # https://github.com/tweepy/tweepy/blob/master/tweepy/api.py
  print api.user_timeline(screen_name="ACTPol_Traffic")
  print api.update_status(status="test")
file:a/servicealerts/index.php -> file:b/servicealerts/index.php
<?php <?php
include ('../include/common.inc.php'); include ('../include/common.inc.php');
   
include_header("Service Alerts", "index"); include_header("Service Alerts", "index");
echo '<ul data-role="listview" data-theme="e" data-groupingtheme="e">';  
   
if ($_SESSION['authed'] == true) { if ($_SESSION['authed'] == true) {
echo' <li><a href="servicealert_editor.php"><h3>servicealert_editor</h3> echo '<ul data-role="listview" data-theme="e" data-groupingtheme="e">
<p>servicealert_editor</p></a></li>'; <li><a href="servicealert_editor.php"><h3>servicealert_editor</h3>
} <p>servicealert_editor</p></a></li>
echo' <li><a href="servicealert_viewer.php"><h3>Service Alert Viewer</h3> </ul>';
<p>Browse current network alerts</p></a></li&