From: Maxious Date: Fri, 21 Dec 2012 01:45:25 +0000 Subject: getfile and convo gen X-Git-Url: https://maxious.lambdacomplex.org/git/?p=scannr.git&a=commitdiff&h=12efdbf1954b5a0722cc7fa02879c007226404a8 --- getfile and convo gen --- --- a/calls.json.php +++ b/calls.json.php @@ -3,10 +3,10 @@ function getTGIDValuesByHour($TGID, $timeFrom, $timeTo) { global $conn; $sth = $conn->prepare( 'select tgid, min(call_timestamp) as time, count(*), min(length), max(length), avg(length), stddev(length) from recordings +where call_timestamp between to_timestamp(?) and to_timestamp(?) group by tgid, date_trunc(\'hour\', call_timestamp) order by time'); - $sth->execute( ); - //Array($TGID, $timeFrom, $timeTo) + $sth->execute(Array($timeFrom, $timeTo)); return $sth->fetchAll(); --- a/generateConvos.php +++ b/generateConvos.php @@ -8,18 +8,26 @@ $convo = Array(); foreach ($recordings as $i => $recording) { - if (count($convo) > 0 && strcasecmp($convo[count($convos)]['tgid'], $recording['tgid']) != 0) { + if (count($convo) > 0) { + echo "
".strcasecmp($convos[count($convos)-1][0]['call_timestamp'],$recording['call_timestamp']); + if (abs(strcasecmp($convos[count($convos)-1][0]['call_timestamp'],$recording['call_timestamp'])) > 2) { + echo " ".$convos[count($convos)-1][0]['call_timestamp']." ".$recording['call_timestamp']; + } + if (strcasecmp($convos[count($convos)-1][0]['tgid'], $recording['tgid']) != 0 ) { $convos[] = $convo; $convo = Array(); + } } ; //print_r($recording); $convo[] = $recording; //print_r($convo); - echo "
\n"; + //echo "
\n"; } foreach ($convos as $i => $convo) { - print_r($convo); - echo "
\n"; + foreach($convo as $recording) { + echo $recording['filename']." , "; + } + echo "
\n"; } ?> --- /dev/null +++ b/getfile.php @@ -1,1 +1,20 @@ + --- a/scannr.py +++ b/scannr.py @@ -12,62 +12,104 @@ #python -m serial.tools.miniterm -p COM20 -e -b 115200 --cr import psycopg2 import csv -import sys,os +import sys +import os sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', 'pynma')) import pynma filename = "demo.wav" +last_call = (None, None, None) MIN_LENGTH = 90000 +lock = threading.RLock() -def worker(filename, length): +def get_call(): + global lock + with lock: + ser.write("GLG\r") + line = ser.readline() # read a '\n' terminated line + print line + reader = csv.reader([line]) + for row in reader: + #GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0 + #,NONE,NONE,NONE + if (row[0] != 'GLG' or row[1] == ''): + print "uh oh" + return (None, None, None) + if (row[1] != ''): + tgid = row[1] + #nma.push("scannr", "ping", filename, "http://www.google.com") + tgname = row[7] + sitename = row[5] + return (tgid, tgname, sitename) + +def log_recording(tgid, tgname, sitename, filename, length): + cur = conn.cursor() + cur.execute("INSERT INTO recordings \ + (tgid, tgname, sitename, filename, length)\ + VALUES (%s, %s, %s, %s, %s)",( tgid, tgname, sitename, filename, length)) + conn.commit() + cur.close() + + +def tgid_worker(): + global last_call + last_call = get_call() + + +def save_worker(filename, length): + global last_call """thread worker function http://www.doughellmann.com/PyMOTW/threading/ https://github.com/uskr/pynma ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k test.3gp -http://stackoverflow.com/questions/2559746/getting-error-while-converting-wav-to-amr-using-ffmpeg +http://stackoverflow.com/questions/2559746/getting-error-while-converting- +wav-to-amr-using-ffmpeg """ - print 'Worker for '+filename - ser.write("GLG\r") - line = ser.readline() # read a '\n' terminated line - print line - reader = csv.reader([line]) - for row in reader: - #GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0,NONE,NONE,NONE - if (row[0] != 'GLG'): - print "uh oh" - if (row[1] != ''): - tgid = row[1] - #nma.push("scannr", "ping", filename, "http://www.google.com") - tgname = row[7] - sitename = row[5] - """http://initd.org/psycopg/docs/usage.html""" - cur = conn.cursor() - cur.execute("INSERT INTO recordings (filename,tgid,tgname,sitename,length) VALUES (%s, %s,%s, %s, %s)",(filename,tgid,tgname,sitename, length)) - conn.commit() - cur.close() + print 'Worker for ' + filename + (oldtgid, oldtgname, oldsitename) = last_call + (tgid, tgname, sitename) = get_call() + if oldtgid == tgid: + if tgid == None or tgid == '': + print filename + " has no TGID" else: - print filename+" has no TGID" + log_recording(tgid, tgname, sitename, filename, length) + else: + if tgid == None or tgid == '': + print filename + " has no TGID" + else: + log_recording(tgid, tgname, sitename, filename, length) + if oldtgid == None or oldtgid == '': + print filename + " has no old TGID" + else: + log_recording(oldtgid, oldtgname, oldsitename, filename, length) + return - return def filenameMaker(): global filename filename = date.today().isoformat()+'-'+str(time.time())+'-demo.wav' -def record_to_async_file(): - "Records from the microphone and outputs the resulting data to `path`" +def do_record(): sample_width, data = snd.record() - print str(len(data)) + thr = threading.Thread(target=record_to_async_file, args=(sample_width,data,filename)) + thr.start() + + +def record_to_async_file(sample_width,data,filename): + "Records from the microphone and outputs the resulting data to `filename`" + print "Recording complete" if len(data) > MIN_LENGTH: + print "Recording being saved..." + dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data)) + print str(len(data)) data = snd.pack('<' + ('h'*len(data)), *data) path = "./data/"+filename - dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data)) wf = wave.open(path, 'wb') - wf.setnchannels(1) + wf.setnchannels(2) wf.setsampwidth(sample_width) wf.setframerate(snd.RATE) wf.writeframes(data) @@ -77,20 +119,23 @@ del data dispatcher.connect( filenameMaker, signal='SND_STARTED', sender=dispatcher.Any ) -dispatcher.connect( worker, signal='FILE_CREATED', sender=dispatcher.Any ) +dispatcher.connect( tgid_worker, signal='SND_STARTED', sender=dispatcher.Any ) +dispatcher.connect( save_worker, signal='FILE_CREATED', sender=dispatcher.Any ) print "Opening serial port..." if sys.platform.startswith('darwin'): - ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1) + ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1) elif sys.platform.startswith('win32'): - ser = serial.Serial('COM20', 112500, timeout=1) + ser = serial.Serial('COM20', 112500, timeout=1) print "Loading notifymyandroid..." -nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f") +nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f") print "Connecting database..." conn = psycopg2.connect("dbname=scannr user=postgres password=snmc") print "Scannr started." while True: print "ready to record again" - record_to_async_file() + do_record() ser.close() + + --- a/snd.py +++ b/snd.py @@ -1,15 +1,12 @@ -""" Record a few seconds of audio and save to a WAVE file. +""" Record a few seconds of audio and save to a WAVE file. Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593 """ import pyaudio import wave import sys -import audioop # http://docs.python.org/library/audioop -from os.path import exists from array import array from struct import unpack, pack -import threading from pydispatch import dispatcher THRESHOLD = 500 @@ -17,11 +14,11 @@ FORMAT = pyaudio.paInt16 RATE = 44100 if sys.platform.startswith('darwin'): - CHANNELS = 2 + CHANNELS = 2 elif sys.platform.startswith('win32'): - CHANNELS = 1 + CHANNELS = 1 -MAX_SILENT = 30 +MAX_SILENT = 80 def is_silent(L): "Returns `True` if below the 'silent' threshold" @@ -72,16 +69,16 @@ def record(): """ - Record a word or words from the microphone and + Record a word or words from the microphone and return the data as an array of signed shorts. - Normalizes the audio, trims silence from the - start and end, and pads with 0.5 seconds of - blank sound to make sure VLC et al can play + Normalizes the audio, trims silence from the + start and end, and pads with 0.5 seconds of + blank sound to make sure VLC et al can play it without getting chopped off. """ p = pyaudio.PyAudio() - stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, + stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK_SIZE) @@ -91,7 +88,12 @@ LRtn = array('h') while 1: - data = stream.read(CHUNK_SIZE) + try: + data = stream.read(CHUNK_SIZE) + except IOError as ex: + if ex[1] != pyaudio.paInputOverflowed: + raise + data = '\x00' * CHUNK_SIZE L = unpack('<' + ('h'*(len(data)/2)), data) # little endian, signed short L = array('h', L) @@ -100,11 +102,13 @@ if silent and snd_started: num_silent += 1 - print num_silent + #print num_silent elif not silent and not snd_started: dispatcher.send( signal='SND_STARTED') snd_started = True print snd_started + if snd_started and not silent: + num_silent = 0 if snd_started: LRtn.extend(L) if snd_started and num_silent > MAX_SILENT: @@ -126,7 +130,7 @@ data = pack('<' + ('h'*len(data)), *data) wf = wave.open(path, 'wb') - wf.setnchannels(1) + wf.setnchannels(CHANNELS) wf.setsampwidth(sample_width) wf.setframerate(RATE) wf.writeframes(data) --- /dev/null +++ b/test.py @@ -1,1 +1,41 @@ +"""PyAudio example: Record a few seconds of audio and save to a WAVE file.""" +import pyaudio +import wave + +CHUNK = 1024 +FORMAT = pyaudio.paInt16 +CHANNELS = 2 +RATE = 44100 +RECORD_SECONDS = 5 +WAVE_OUTPUT_FILENAME = "output.wav" + +p = pyaudio.PyAudio() + +stream = p.open(format=FORMAT, + channels=CHANNELS, + rate=RATE, + input=True, + frames_per_buffer=CHUNK) + +print("* recording") + +frames = [] + +for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)): + data = stream.read(CHUNK) + frames.append(data) + +print("* done recording") + +stream.stop_stream() +stream.close() +p.terminate() + +wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb') +wf.setnchannels(CHANNELS) +wf.setsampwidth(p.get_sample_size(FORMAT)) +wf.setframerate(RATE) +wf.writeframes(b''.join(frames)) +wf.close() + --- /dev/null +++ b/test2.py @@ -1,1 +1,121 @@ +from os.path import exists +from array import array +from struct import unpack, pack +import pyaudio +import wave + +THRESHOLD = 500 +CHUNK_SIZE = 1024 +FORMAT = pyaudio.paInt16 +RATE = 44100 + +def is_silent(L): + "Returns `True` if below the 'silent' threshold" + return max(L) < THRESHOLD + +def normalize(L): + "Average the volume out" + MAXIMUM = 16384 + times = float(MAXIMUM)/max(abs(i) for i in L) + + LRtn = array('h') + for i in L: + LRtn.append(int(i*times)) + return LRtn + +def trim(L): + "Trim the blank spots at the start and end" + def _trim(L): + snd_started = False + LRtn = array('h') + + for i in L: + if not snd_started and abs(i)>THRESHOLD: + snd_started = True + LRtn.append(i) + + elif snd_started: + LRtn.append(i) + return LRtn + + # Trim to the left + L = _trim(L) + + # Trim to the right + L.reverse() + L = _trim(L) + L.reverse() + return L + +def add_silence(L, seconds): + "Add silence to the start and end of `L` of length `seconds` (float)" + LRtn = array('h', [0 for i in xrange(int(seconds*RATE))]) + LRtn.extend(L) + LRtn.extend([0 for i in xrange(int(seconds*RATE))]) + return LRtn + +def record(): + """ + Record a word or words from the microphone and + return the data as an array of signed shorts. + + Normalizes the audio, trims silence from the + start and end, and pads with 0.5 seconds of + blank sound to make sure VLC et al can play + it without getting chopped off. + """ + p = pyaudio.PyAudio() + stream = p.open(format=FORMAT, channels=2, rate=RATE, + input=True, + frames_per_buffer=CHUNK_SIZE) + + num_silent = 0 + snd_started = False + + LRtn = array('h') + + while 1: + data = stream.read(CHUNK_SIZE) + L = unpack('<' + ('h'*(len(data)/2)), data) # little endian, signed short + L = array('h', L) + LRtn.extend(L) + + silent = is_silent(L) + #print silent, num_silent, L[:10] + + if silent and snd_started: + num_silent += 1 + elif not silent and not snd_started: + snd_started = True + + if snd_started and num_silent > 30: + break + + sample_width = p.get_sample_size(FORMAT) + stream.stop_stream() + stream.close() + p.terminate() + + LRtn = normalize(LRtn) + LRtn = trim(LRtn) + LRtn = add_silence(LRtn, 0.5) + return sample_width, LRtn + +def record_to_file(path): + "Records from the microphone and outputs the resulting data to `path`" + sample_width, data = record() + data = pack('<' + ('h'*len(data)), *data) + + wf = wave.open(path, 'wb') + wf.setnchannels(2) + wf.setsampwidth(sample_width) + wf.setframerate(RATE) + wf.writeframes(data) + wf.close() + +if __name__ == '__main__': + print("please speak a word into the microphone") + record_to_file('demo.wav') + print("done - result written to demo.wav") + --- a/viewcalls.php +++ b/viewcalls.php @@ -1,7 +1,16 @@ prepare('select distinct date_trunc(\'day\', call_timestamp) as rdate from recordings order by rdate'); + + $sth->execute(); + foreach( $sth->fetchAll() as $row) { +echo ''.$row['rdate'].'
'; +} + ?>
@@ -48,7 +57,7 @@ Flotr.EventAdapter.observe(placeholder, 'flotr:click', function () { drawGraph(); }); - getData('','',''); + getData('','',''); }); @@ -91,3 +100,4 @@ +