gitphp 0.2.9.1 :: scannr.git/commitdiff

parallelise serial port usage and I/O for writing sounds files

2 files changed: (show all)
scannr.py
snd.py

file:a/scannr.py -> file:b/scannr.py

import logging	import logging
logging.basicConfig(level=logging.DEBUG,	logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s\t%(levelname)s\t%(message)s')	format='%(asctime)s\t%(levelname)s\t%(message)s')

import snd	import snd
import time	import time
from datetime import date	from datetime import date
import threading	import threading
from pydispatch import dispatcher	from pydispatch import dispatcher
import wave	import wave
import serial	import serial
#python -m serial.tools.miniterm -p COM20 -e -b 115200 --cr	#python -m serial.tools.miniterm -p COM20 -e -b 115200 --cr
import psycopg2	import psycopg2
import csv	import csv
import sys,os	import sys
	import os


sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', 'pynma'))	sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', 'pynma'))
import pynma	import pynma

filename = "demo.wav"	filename = "demo.wav"
	last_call = (None, None, None)
MIN_LENGTH = 90000	MIN_LENGTH = 90000
	lock = threading.RLock()


def worker(filename, length):	def get_call():
	global lock
	with lock:
	ser.write("GLG\r")
	line = ser.readline() # read a '\n' terminated line
	print line
	reader = csv.reader([line])
	for row in reader:
	#GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0
	#,NONE,NONE,NONE
	if (row[0] != 'GLG' or row[1] == ''):
	print "uh oh"
	return (None, None, None)
	if (row[1] != ''):
	tgid = row[1]
	#nma.push("scannr", "ping", filename, "http://www.google.com")
	tgname = row[7]
	sitename = row[5]
	return (tgid, tgname, sitename)


	def log_recording(tgid, tgname, sitename, filename, length):
	cur = conn.cursor()
	cur.execute("INSERT INTO recordings \
	(tgid, tgname, sitename, filename, length)\
	VALUES (%s, %s, %s, %s, %s)",( tgid, tgname, sitename, filename, length))
	conn.commit()
	cur.close()


	def tgid_worker():
	global last_call
	last_call = get_call()


	def save_worker(filename, length):
	global last_call
"""thread worker function	"""thread worker function
http://www.doughellmann.com/PyMOTW/threading/	http://www.doughellmann.com/PyMOTW/threading/
https://github.com/uskr/pynma	https://github.com/uskr/pynma
ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k test.3gp	ffmpeg -i 2012-09-29-1348911268.34-demo.wav -ar 8000 -ab 4.75k test.3gp
http://stackoverflow.com/questions/2559746/getting-error-while-converting-wav-to-amr-using-ffmpeg	http://stackoverflow.com/questions/2559746/getting-error-while-converting-
	wav-to-amr-using-ffmpeg
"""	"""
print 'Worker for '+filename	print 'Worker for ' + filename
ser.write("GLG\r")	(oldtgid, oldtgname, oldsitename) = last_call
line = ser.readline() # read a '\n' terminated line	(tgid, tgname, sitename) = get_call()
print line	if oldtgid == tgid:
reader = csv.reader([line])	if tgid == None or tgid == '':
for row in reader:	print filename + " has no TGID"
#GLG,40078,NFM,0,0,CanberraBlackMnt,AustralianCapita,SES Ops 1,1,0,NONE,NONE,NONE
if (row[0] != 'GLG'):
print "uh oh"
if (row[1] != ''):
tgid = row[1]
#nma.push("scannr", "ping", filename, "http://www.google.com")
tgname = row[7]
sitename = row[5]
"""http://initd.org/psycopg/docs/usage.html"""
cur = conn.cursor()
cur.execute("INSERT INTO recordings (filename,tgid,tgname,sitename,length) VALUES (%s, %s,%s, %s, %s)",(filename,tgid,tgname,sitename, length))
conn.commit()
cur.close()
else:	else:
print filename+" has no TGID"	log_recording(tgid, tgname, sitename, filename, length)
	else:
	if tgid == None or tgid == '':
	print filename + " has no TGID"
	else:
	log_recording(tgid, tgname, sitename, filename, length)
	if oldtgid == None or oldtgid == '':
	print filename + " has no old TGID"
	else:
	log_recording(oldtgid, oldtgname, oldsitename, filename, length)
	return

return

def filenameMaker():	def filenameMaker():
global filename	global filename
filename = date.today().isoformat()+'-'+str(time.time())+'-demo.wav'	filename = date.today().isoformat()+'-'+str(time.time())+'-demo.wav'

def record_to_async_file():	def do_record():
"Records from the microphone and outputs the resulting data to `path`"
sample_width, data = snd.record()	sample_width, data = snd.record()
print str(len(data))	thr = threading.Thread(target=record_to_async_file, args=(sample_width,data,filename))
	thr.start()


	def record_to_async_file(sample_width,data,filename):
	"Records from the microphone and outputs the resulting data to `filename`"
	print "Recording complete"
if len(data) > MIN_LENGTH:	if len(data) > MIN_LENGTH:
	print "Recording being saved..."
	dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data))
	print str(len(data))
data = snd.pack('<' + ('h'len(data)), data)	data = snd.pack('<' + ('h'len(data)), data)
path = "./data/"+filename	path = "./data/"+filename
dispatcher.send( signal='FILE_CREATED', sender=filename, filename=filename, length=len(data))
wf = wave.open(path, 'wb')	wf = wave.open(path, 'wb')
wf.setnchannels(2)	wf.setnchannels(2)
wf.setsampwidth(sample_width)	wf.setsampwidth(sample_width)
wf.setframerate(snd.RATE)	wf.setframerate(snd.RATE)
wf.writeframes(data)	wf.writeframes(data)
wf.close()	wf.close()
del wf	del wf
print("done - result "+str(len(data))+" frames written to "+path)	print("done - result "+str(len(data))+" frames written to "+path)
del data	del data

dispatcher.connect( filenameMaker, signal='SND_STARTED', sender=dispatcher.Any )	dispatcher.connect( filenameMaker, signal='SND_STARTED', sender=dispatcher.Any )
dispatcher.connect( worker, signal='FILE_CREATED', sender=dispatcher.Any )	dispatcher.connect( tgid_worker, signal='SND_STARTED', sender=dispatcher.Any )
	dispatcher.connect( save_worker, signal='FILE_CREATED', sender=dispatcher.Any )

print "Opening serial port..."	print "Opening serial port..."
if sys.platform.startswith('darwin'):	if sys.platform.startswith('darwin'):
ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1)	ser = serial.Serial('/dev/tty.usbserial-FTB3VL83', 112500, timeout=1)
elif sys.platform.startswith('win32'):	elif sys.platform.startswith('win32'):
ser = serial.Serial('COM20', 112500, timeout=1)	ser = serial.Serial('COM20', 112500, timeout=1)
print "Loading notifymyandroid..."	print "Loading notifymyandroid..."
nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f")	nma = pynma.PyNMA( "a6f50f76119eda33befe4325b4b9e1dd25eef7bad2868e4f")
print "Connecting database..."	print "Connecting database..."
conn = psycopg2.connect("dbname=scannr user=postgres password=snmc")	conn = psycopg2.connect("dbname=scannr user=postgres password=snmc")
print "Scannr started."	print "Scannr started."
while True:	while True:
print "ready to record again"	print "ready to record again"
record_to_async_file()	do_record()
ser.close()	ser.close()

file:a/snd.py -> file:b/snd.py

""" Record a few seconds of audio and save to a WAVE file.	""" Record a few seconds of audio and save to a WAVE file.
Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593	Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593
"""	"""

import pyaudio	import pyaudio
import wave	import wave
import sys	import sys
import audioop # http://docs.python.org/library/audioop
from os.path import exists
from array import array	from array import array
from struct import unpack, pack	from struct import unpack, pack
import threading
from pydispatch import dispatcher	from pydispatch import dispatcher

THRESHOLD = 500	THRESHOLD = 500
CHUNK_SIZE = 1024	CHUNK_SIZE = 1024
FORMAT = pyaudio.paInt16	FORMAT = pyaudio.paInt16
RATE = 44100	RATE = 44100
if sys.platform.startswith('darwin'):	if sys.platform.startswith('darwin'):
CHANNELS = 2	CHANNELS = 2
elif sys.platform.startswith('win32'):	elif sys.platform.startswith('win32'):
CHANNELS = 1	CHANNELS = 1

MAX_SILENT = 60	MAX_SILENT = 80

def is_silent(L):	def is_silent(L):
"Returns `True` if below the 'silent' threshold"	"Returns `True` if below the 'silent' threshold"
"print max(L)"	"print max(L)"
"print max(L) < THRESHOLD"	"print max(L) < THRESHOLD"
return max(L) < THRESHOLD	return max(L) < THRESHOLD

def normalize(L):	def normalize(L):
"Average the volume out"	"Average the volume out"
MAXIMUM = 16384	MAXIMUM = 16384
times = float(MAXIMUM)/max(abs(i) for i in L)	times = float(MAXIMUM)/max(abs(i) for i in L)

LRtn = array('h')	LRtn = array('h')
for i in L:	for i in L:
LRtn.append(int(i*times))	LRtn.append(int(i*times))
return LRtn	return LRtn

def trim(L):	def trim(L):
"Trim the blank spots at the start and end"	"Trim the blank spots at the start and end"
def _trim(L):	def _trim(L):
snd_started = False	snd_started = False
LRtn = array('h')	LRtn = array('h')

for i in L:	for i in L:
if not snd_started and abs(i)>THRESHOLD:	if not snd_started and abs(i)>THRESHOLD:
snd_started = True	snd_started = True
LRtn.append(i)	LRtn.append(i)

elif snd_started:	elif snd_started:
LRtn.append(i)	LRtn.append(i)
return LRtn	return LRtn

# Trim to the left	# Trim to the left
L = _trim(L)	L = _trim(L)

# Trim to the right	# Trim to the right
L.reverse()	L.reverse()
L = _trim(L)	L = _trim(L)
L.reverse()	L.reverse()
return L	return L

def add_silence(L, seconds):	def add_silence(L, seconds):
"Add silence to the start and end of `L` of length `seconds` (float)"	"Add silence to the start and end of `L` of length `seconds` (float)"
LRtn = array('h', [0 for i in xrange(int(seconds*RATE))])	LRtn = array('h', [0 for i in xrange(int(seconds*RATE))])
LRtn.extend(L)	LRtn.extend(L)
LRtn.extend([0 for i in xrange(int(seconds*RATE))])	LRtn.extend([0 for i in xrange(int(seconds*RATE))])
return LRtn	return LRtn

def record():	def record():
"""	"""
Record a word or words from the microphone and	Record a word or words from the microphone and
return the data as an array of signed shorts.	return the data as an array of signed shorts.

Normalizes the audio, trims silence from the	Normalizes the audio, trims silence from the
start and end, and pads with 0.5 seconds of	start and end, and pads with 0.5 seconds of
blank sound to make sure VLC et al can play	blank sound to make sure VLC et al can play
it without getting chopped off.	it without getting chopped off.
"""	"""
p = pyaudio.PyAudio()	p = pyaudio.PyAudio()
stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE,	stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE,
input=True,	input=True,
frames_per_buffer=CHUNK_SIZE)	frames_per_buffer=CHUNK_SIZE)

num_silent = 0	num_silent = 0
snd_started = False	snd_started = False

LRtn = array('h')	LRtn = array('h')

while 1:	while 1:
try:	try:
data = stream.read(CHUNK_SIZE)	data = stream.read(CHUNK_SIZE)
except IOError as ex:	except IOError as ex:
if ex[1] != pyaudio.paInputOverflowed:	if ex[1] != pyaudio.paInputOverflowed:
raise	raise
data = '\x00' * CHUNK_SIZE	data = '\x00' * CHUNK_SIZE
L = unpack('<' + ('h'*(len(data)/2)), data) # little endian, signed short	L = unpack('<' + ('h'*(len(data)/2)), data) # little endian, signed short
L = array('h', L)	L = array('h', L)

silent = is_silent(L)	silent = is_silent(L)
#print silent, num_silent, L[:10]	#print silent, num_silent, L[:10]

if silent and snd_started:	if silent and snd_started:
num_silent += 1	num_silent += 1
#print num_silent	#print num_silent
elif not silent and not snd_started:	elif not silent and not snd_started:
dispatcher.send( signal='SND_STARTED')	dispatcher.send( signal='SND_STARTED')
snd_started = True	snd_started = True
print snd_started	print snd_started
	if snd_started and not silent:
	num_silent = 0
if snd_started:	if snd_started:
LRtn.extend(L)	LRtn.extend(L)
if snd_started and num_silent > MAX_SILENT:	if snd_started and num_silent > MAX_SILENT:
break	break

sample_width = p.get_sample_size(FORMAT)	sample_width = p.get_sample_size(FORMAT)
stream.stop_stream()	stream.stop_stream()
stream.close()	stream.close()
p.terminate()	p.terminate()

LRtn = normalize(LRtn)	LRtn = normalize(LRtn)
LRtn = trim(LRtn)	LRtn = trim(LRtn)
LRtn = add_silence(LRtn, 0.5)	LRtn = add_silence(LRtn, 0.5)
return sample_width, LRtn	return sample_width, LRtn

def record_to_file(path):	def record_to_file(path):
"Records from the microphone and outputs the resulting data to `path`"	"Records from the microphone and outputs the resulting data to `path`"
sample_width, data = record()	sample_width, data = record()
data = pack('<' + ('h'len(data)), data)	data = pack('<' + ('h'len(data)), data)

wf = wave.open(path, 'wb')	wf = wave.open(path, 'wb')
wf.setnchannels(CHANNELS)	wf.setnchannels(CHANNELS)
wf.setsampwidth(sample_width)	wf.setsampwidth(sample_width)
wf.setframerate(RATE)	wf.setframerate(RATE)
wf.writeframes(data)	wf.writeframes(data)
wf.close()	wf.close()
print("done - result written to "+path)	print("done - result written to "+path)
del data	del data




if __name__ == '__main__':	if __name__ == '__main__':
filename = 'demo.wav'	filename = 'demo.wav'
record_to_file(filename)	record_to_file(filename)
print("done - result written to "+filename)	print("done - result written to "+filename)