getfile and convo gen
getfile and convo gen

<?php <?php
include('common.inc.php'); include('common.inc.php');
function getTGIDValuesByHour($TGID, $timeFrom, $timeTo) { function getTGIDValuesByHour($TGID, $timeFrom, $timeTo) {
global $conn; global $conn;
$sth = $conn->prepare( 'select tgid, min(call_timestamp) as time, count(*), min(length), max(length), avg(length), stddev(length) from recordings $sth = $conn->prepare( 'select tgid, min(call_timestamp) as time, count(*), min(length), max(length), avg(length), stddev(length) from recordings
  where call_timestamp between to_timestamp(?) and to_timestamp(?)
group by tgid, date_trunc(\'hour\', call_timestamp) order by time'); group by tgid, date_trunc(\'hour\', call_timestamp) order by time');
   
$sth->execute( ); $sth->execute(Array($timeFrom, $timeTo));
//Array($TGID, $timeFrom, $timeTo)  
return $sth->fetchAll(); return $sth->fetchAll();
   
   
} }
   
function getTGIDValuesByDay($TGID, $dayFrom, $dayTo) { function getTGIDValuesByDay($TGID, $dayFrom, $dayTo) {
global $conn; global $conn;
$sth = $conn->prepare('select min(time) as time, min(value), max(value), avg(value), stddev(value) from sensor_values where sensor_id = ? $sth = $conn->prepare('select min(time) as time, min(value), max(value), avg(value), stddev(value) from sensor_values where sensor_id = ?
group by sensor_id, date_trunc(\'day\', time) order by time'); group by sensor_id, date_trunc(\'day\', time) order by time');
   
$sth->execute( Array($TGID)); $sth->execute( Array($TGID));
return $sth->fetchAll(); return $sth->fetchAll();
} }
function getTGIDDataYears($TGID, $timeFrom, $timeTo) { function getTGIDDataYears($TGID, $timeFrom, $timeTo) {
global $conn; global $conn;
$sth = $conn->prepare("select distinct extract('year' from call_timestamp) as year from recordings where tgid = ? order by year"); $sth = $conn->prepare("select distinct extract('year' from call_timestamp) as year from recordings where tgid = ? order by year");
   
$sth->execute(Array($TGID)); $sth->execute(Array($TGID));
return $sth->fetchAll(); return $sth->fetchAll();
} }
   
function getTGIDDataMonths($TGID, $timeFrom, $timeTo) { function getTGIDDataMonths($TGID, $timeFrom, $timeTo) {
global $conn; global $conn;
$sth = $conn->prepare("select distinct extract('month' from call_timestamp) as month, extract('year' from call_timestamp) as year from recordings where tgid = ? order by year, month"); $sth = $conn->prepare("select distinct extract('month' from call_timestamp) as month, extract('year' from call_timestamp) as year from recordings where tgid = ? order by year, month");
   
$sth->execute(Array($TGID)); $sth->execute(Array($TGID));
return $sth->fetchAll(); return $sth->fetchAll();
} }
   
function getTGIDDataDays($TGID, $timeFrom, $timeTo) { function getTGIDDataDays($TGID, $timeFrom, $timeTo) {
global $conn; global $conn;
$sth = $conn->prepare("select distinct extract('day' from call_timestamp) as day, extract('month' from call_timestamp) as month, extract('year' from call_timestamp) as year from recordings where tgid = ? order by year,month,day"); $sth = $conn->prepare("select distinct extract('day' from call_timestamp) as day, extract('month' from call_timestamp) as month, extract('year' from call_timestamp) as year from recordings where tgid = ? order by year,month,day");
   
   
$sth->execute(Array($TGID)); $sth->execute(Array($TGID));
return $sth->fetchAll(); return $sth->fetchAll();
} }
$action = (isset($_REQUEST['action']) ? $_REQUEST['action'] : ''); $action = (isset($_REQUEST['action']) ? $_REQUEST['action'] : '');
$TGID = (isset($_REQUEST['tgid']) ? $_REQUEST['tgid'] : ''); $TGID = (isset($_REQUEST['tgid']) ? $_REQUEST['tgid'] : '');
$timefrom = (isset($_REQUEST['from']) ? $_REQUEST['from'] : ''); $timefrom = (isset($_REQUEST['from']) ? $_REQUEST['from'] : '');
$timeto = (isset($_REQUEST['to']) ? $_REQUEST['to'] : ''); $timeto = (isset($_REQUEST['to']) ? $_REQUEST['to'] : '');
   
if ($action == "data_description") { if ($action == "data_description") {
$timefrom = strtotime($timefrom); $timefrom = strtotime($timefrom);
$timeto = strtotime($timeto); $timeto = strtotime($timeto);
$years = getTGIDDataYears($TGID, $timefrom, $timeto); $years = getTGIDDataYears($TGID, $timefrom, $timeto);
   
$months = getTGIDDataMonths($TGID, $timefrom, $timeto); $months = getTGIDDataMonths($TGID, $timefrom, $timeto);
$days = getTGIDDataDays($TGID, $timefrom, $timeto); $days = getTGIDDataDays($TGID, $timefrom, $timeto);
   
echo json_encode(Array("years" => $years, "months" => $months, "days" => $days echo json_encode(Array("years" => $years, "months" => $months, "days" => $days
)); ));
} }
   
   
if (strpos($action,"graph") !== false) { if (strpos($action,"graph") !== false) {
$values = getTGIDValuesByHour($TGID, $timefrom, $timeto); $values = getTGIDValuesByHour($TGID, $timefrom, $timeto);
$label = $TGID; $label = $TGID;
$data = Array(); $data = Array();
$tzoffset = get_timezone_offset("UTC"); $tzoffset = get_timezone_offset("UTC");
foreach ($values as $value) { foreach ($values as $value) {
if ($action == "graphlength") { if ($action == "graphlength") {
$data[$value['tgid']][] = Array((strtotime($value['time']) + $tzoffset) * 1000, intval($value['avg'])); $data[$value['tgid']][] = Array((strtotime($value['time']) + $tzoffset) * 1000, intval($value['avg']));
} else if ($action == "graphcount") { } else if ($action == "graphcount") {
$data[$value['tgid']][] = Array((strtotime($value['time']) + $tzoffset) * 1000, intval($value['count'])); $data[$value['tgid']][] = Array((strtotime($value['time']) + $tzoffset) * 1000, intval($value['count']));
} }
} }
echo json_encode(Array("label" => $label, "data" => $data, echo json_encode(Array("label" => $label, "data" => $data,
"previous" => Array( "previous" => Array(
"from" => $timefrom - (24 * 60 * 60), "from" => $timefrom - (24 * 60 * 60),
"to" => $timefrom) "to" => $timefrom)
, ,
"next" => Array( "next" => Array(
"to" => $timeto + (24 * 60 * 60), "to" => $timeto + (24 * 60 * 60),
"from" => $timeto) "from" => $timeto)
) )
); );
} }
   
   
   
?> ?>
   
<?php <?php
include('common.inc.php'); include('common.inc.php');
$sth = $conn->prepare( 'select * from recordings limit 100;'); $sth = $conn->prepare( 'select * from recordings limit 100;');
   
$sth->execute( ); $sth->execute( );
$recordings = $sth->fetchAll(); $recordings = $sth->fetchAll();
$convos = Array(); $convos = Array();
$convo = Array(); $convo = Array();
foreach ($recordings as $i => $recording) { foreach ($recordings as $i => $recording) {
   
if (count($convo) > 0 && strcasecmp($convo[count($convos)]['tgid'], $recording['tgid']) != 0) { if (count($convo) > 0) {
  echo "<br> ".strcasecmp($convos[count($convos)-1][0]['call_timestamp'],$recording['call_timestamp']);
  if (abs(strcasecmp($convos[count($convos)-1][0]['call_timestamp'],$recording['call_timestamp'])) > 2) {
  echo " ".$convos[count($convos)-1][0]['call_timestamp']." ".$recording['call_timestamp'];
  }
  if (strcasecmp($convos[count($convos)-1][0]['tgid'], $recording['tgid']) != 0 ) {
$convos[] = $convo; $convos[] = $convo;
$convo = Array(); $convo = Array();
  }
} ; } ;
//print_r($recording); //print_r($recording);
$convo[] = $recording; $convo[] = $recording;
//print_r($convo); //print_r($convo);
echo "<br>\n"; //echo "<br>\n";
} }
foreach ($convos as $i => $convo) { foreach ($convos as $i => $convo) {
print_r($convo); foreach($convo as $recording) {
echo "<br>\n"; echo $recording['filename']." , ";
  }
  echo "<br><hr>\n";
} }
?> ?>
   
   
file:b/getfile.php (new)
  <?php
  $reqfile = "path/to/file.3gp";
  $contenttype="audio/3gpp";
 
  if($fn=fopen($reqfile, "rba")){
  header("Content-Type: ".$contenttype);
  header("Content-Length: ".filesize($reqfile));
  header("Last-Modified: " . gmdate("D, d M Y H:i:s") . " GMT");
  header("Pragma: no-cache");
  header("Expires: Mon, 26 Jul 1997 06:00:00 GMT");
  header("Cache-Control: no-store, no-cache, must-revalidate, max-age=0, post-check=0, pre-check=0");
  passthru("ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k -");
  fpassthru($fn);
  fclose($fn);
  }else{
  exit("error....");
  }
  exit();
  ?>
 
file:a/scannr.py -> file:b/scannr.py
import logging import logging
logging.basicConfig(level=logging.DEBUG, logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s\t%(levelname)s\t%(message)s') format='%(asctime)s\t%(levelname)s\t%(message)s')
   
import snd import snd
import time import time
from datetime import date from datetime import date
import threading import threading
from pydispatch import dispatcher from pydispatch import dispatcher
import wave import wave
import serial import serial
#python -m serial.tools.miniterm -p COM20 -e -b 115200 --cr #python -m serial.tools.miniterm -p COM20 -e -b 115200 --cr
import psycopg2 import psycopg2
import csv import csv
import sys,os import sys
  import os
   
   
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', 'pynma')) sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', 'pynma'))
import pynma import pynma
   
filename = "demo.wav" filename = "demo.wav"
  last_call = (None, None, None)
MIN_LENGTH = 90000 MIN_LENGTH = 90000
  lock = threading.RLock()
   
   
def worker(filename, length): def get_call():
  global lock
  with lock:
  ser.write("GLG\r")
  line = ser.readline() # read a '\n' terminated line
  print line
  reader = csv.reader([line])
  for row in reader:
  #GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0
  #,NONE,NONE,NONE
  if (row[0] != 'GLG' or row[1] == ''):
  print "uh oh"
  return (None, None, None)
  if (row[1] != ''):
  tgid = row[1]
  #nma.push("scannr", "ping", filename, "http://www.google.com")
  tgname = row[7]
  sitename = row[5]
  return (tgid, tgname, sitename)
   
   
  def log_recording(tgid, tgname, sitename, filename, length):
  cur = conn.cursor()
  cur.execute("INSERT INTO recordings \
  (tgid, tgname, sitename, filename, length)\
  VALUES (%s, %s, %s, %s, %s)",( tgid, tgname, sitename, filename, length))
  conn.commit()
  cur.close()
   
   
  def tgid_worker():
  global last_call
  last_call = get_call()
   
   
  def save_worker(filename, length):
  global last_call
"""thread worker function """thread worker function
http://www.doughellmann.com/PyMOTW/threading/ http://www.doughellmann.com/PyMOTW/threading/
https://github.com/uskr/pynma https://github.com/uskr/pynma
ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k test.3gp ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k test.3gp
http://stackoverflow.com/questions/2559746/getting-error-while-converting-wav-to-amr-using-ffmpeg http://stackoverflow.com/questions/2559746/getting-error-while-converting-
  wav-to-amr-using-ffmpeg
""" """
print 'Worker for '+filename print 'Worker for ' + filename
ser.write("GLG\r") (oldtgid, oldtgname, oldsitename) = last_call
line = ser.readline() # read a '\n' terminated line (tgid, tgname, sitename) = get_call()
print line if oldtgid == tgid:
reader = csv.reader([line]) if tgid == None or tgid == '':
for row in reader: print filename + " has no TGID"
#GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0,NONE,NONE,NONE  
if (row[0] != 'GLG'):  
print "uh oh"  
if (row[1] != ''):  
tgid = row[1]  
#nma.push("scannr", "ping", filename, "http://www.google.com")  
tgname = row[7]  
sitename = row[5]  
"""http://initd.org/psycopg/docs/usage.html"""  
cur = conn.cursor()  
cur.execute("INSERT INTO recordings (filename,tgid,tgname,sitename,length) VALUES (%s, %s,%s, %s, %s)",(filename,tgid,tgname,sitename, length))  
conn.commit()  
cur.close()  
else: else:
print filename+" has no TGID" log_recording(tgid, tgname, sitename, filename, length)
  else:
  if tgid == None or tgid == '':
  print filename + " has no TGID"
  else:
  log_recording(tgid, tgname, sitename, filename, length)
  if oldtgid == None or oldtgid == '':
  print filename + " has no old TGID"
  else:
  log_recording(oldtgid, oldtgname, oldsitename, filename, length)
  return
   
return  
   
def filenameMaker(): def filenameMaker():
global filename global filename
filename = date.today().isoformat()+'-'+str(time.time())+'-demo.wav' filename = date.today().isoformat()+'-'+str(time.time())+'-demo.wav'
   
def record_to_async_file(): def do_record():
"Records from the microphone and outputs the resulting data to `path`"  
sample_width, data = snd.record() sample_width, data = snd.record()
print str(len(data)) thr = threading.Thread(target=record_to_async_file, args=(sample_width,data,filename))
  thr.start()
   
   
  def record_to_async_file(sample_width,data,filename):
  "Records from the microphone and outputs the resulting data to `filename`"
  print "Recording complete"
if len(data) > MIN_LENGTH: if len(data) > MIN_LENGTH:
  print "Recording being saved..."
  dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data))
  print str(len(data))
data = snd.pack('<' + ('h'*len(data)), *data) data = snd.pack('<' + ('h'*len(data)), *data)
path = "./data/"+filename path = "./data/"+filename
dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data))  
wf = wave.open(path, 'wb') wf = wave.open(path, 'wb')
wf.setnchannels(2) wf.setnchannels(2)
wf.setsampwidth(sample_width) wf.setsampwidth(sample_width)
wf.setframerate(snd.RATE) wf.setframerate(snd.RATE)
wf.writeframes(data) wf.writeframes(data)
wf.close() wf.close()
del wf del wf
print("done - result "+str(len(data))+" frames written to "+path) print("done - result "+str(len(data))+" frames written to "+path)
del data del data
   
dispatcher.connect( filenameMaker, signal='SND_STARTED', sender=dispatcher.Any ) dispatcher.connect( filenameMaker, signal='SND_STARTED', sender=dispatcher.Any )
dispatcher.connect( worker, signal='FILE_CREATED', sender=dispatcher.Any ) dispatcher.connect( tgid_worker, signal='SND_STARTED', sender=dispatcher.Any )
  dispatcher.connect( save_worker, signal='FILE_CREATED', sender=dispatcher.Any )
   
print "Opening serial port..." print "Opening serial port..."
if sys.platform.startswith('darwin'): if sys.platform.startswith('darwin'):
ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1) ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1)
elif sys.platform.startswith('win32'): elif sys.platform.startswith('win32'):
ser = serial.Serial('COM20', 112500, timeout=1) ser = serial.Serial('COM20', 112500, timeout=1)
print "Loading notifymyandroid..." print "Loading notifymyandroid..."
nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f") nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f")
print "Connecting database..." print "Connecting database..."
conn = psycopg2.connect("dbname=scannr user=postgres password=snmc") conn = psycopg2.connect("dbname=scannr user=postgres password=snmc")
print "Scannr started." print "Scannr started."
while True: while True:
print "ready to record again" print "ready to record again"
record_to_async_file() do_record()
ser.close() ser.close()
   
   
   
file:a/snd.py -> file:b/snd.py
""" Record a few seconds of audio and save to a WAVE file. """ Record a few seconds of audio and save to a WAVE file.
Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593 Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593
""" """
   
import pyaudio import pyaudio
import wave import wave
import sys import sys
import audioop # http://docs.python.org/library/audioop  
from os.path import exists  
from array import array from array import array
from struct import unpack, pack from struct import unpack, pack
import threading  
from pydispatch import dispatcher from pydispatch import dispatcher
   
THRESHOLD = 500 THRESHOLD = 500
CHUNK_SIZE = 1024 CHUNK_SIZE = 1024
FORMAT = pyaudio.paInt16 FORMAT = pyaudio.paInt16
RATE = 44100 RATE = 44100
if sys.platform.startswith('darwin'): if sys.platform.startswith('darwin'):
CHANNELS = 2 CHANNELS = 2
elif sys.platform.startswith('win32'): elif sys.platform.startswith('win32'):
CHANNELS = 1 CHANNELS = 1
   
MAX_SILENT = 60 MAX_SILENT = 80
   
def is_silent(L): def is_silent(L):
"Returns `True` if below the 'silent' threshold" "Returns `True` if below the 'silent' threshold"
"print max(L)" "print max(L)"
"print max(L) < THRESHOLD" "print max(L) < THRESHOLD"
return max(L) < THRESHOLD return max(L) < THRESHOLD
   
def normalize(L): def normalize(L):
"Average the volume out" "Average the volume out"
MAXIMUM = 16384 MAXIMUM = 16384
times = float(MAXIMUM)/max(abs(i) for i in L) times = float(MAXIMUM)/max(abs(i) for i in L)
   
LRtn = array('h')