| -- /usr/pgsql-9.1/bin/pg_dump --schema-only scannr | -- /usr/pgsql-9.1/bin/pg_dump --schema-only scannr |
| -- | -- |
| -- PostgreSQL database dump | -- PostgreSQL database dump |
| -- | -- |
| SET statement_timeout = 0; | SET statement_timeout = 0; |
| SET client_encoding = 'UTF8'; | SET client_encoding = 'UTF8'; |
| SET standard_conforming_strings = on; | SET standard_conforming_strings = on; |
| SET check_function_bodies = false; | SET check_function_bodies = false; |
| SET client_min_messages = warning; | SET client_min_messages = warning; |
| -- | -- |
| -- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: | -- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: |
| -- | -- |
| CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; | CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; |
| -- | -- |
| -- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: | -- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: |
| -- | -- |
| COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; | COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; |
| SET search_path = public, pg_catalog; | SET search_path = public, pg_catalog; |
| SET default_tablespace = ''; | SET default_tablespace = ''; |
| SET default_with_oids = false; | SET default_with_oids = false; |
| -- | -- |
| -- Name: recordings; Type: TABLE; Schema: public; Owner: postgres; Tablespace: | -- Name: recordings; Type: TABLE; Schema: public; Owner: postgres; Tablespace: |
| -- | -- |
| CREATE TABLE recordings ( | CREATE TABLE recordings ( |
| filename text NOT NULL, | filename text NOT NULL, |
| tgid text, | tgid text, |
| tgname text, | tgname text, |
| sitename text, | sitename text, |
| call_timestamp timestamp with time zone DEFAULT now(), | call_timestamp timestamp with time zone DEFAULT now(), |
| length integer | length integer |
| ); | ); |
| ALTER TABLE public.recordings OWNER TO postgres; | ALTER TABLE public.recordings OWNER TO postgres; |
| -- | -- |
| -- Name: tgids; Type: TABLE; Schema: public; Owner: postgres; Tablespace: | -- Name: tgids; Type: TABLE; Schema: public; Owner: postgres; Tablespace: |
| -- | -- |
| CREATE TABLE tgids ( | CREATE TABLE tgids ( |
| tgid text NOT NULL, | tgid text NOT NULL, |
| subfleet smallint, | subfleet smallint, |
| alpha_tag text NOT NULL, | alpha_tag text NOT NULL, |
| mode character(1) DEFAULT 'D'::bpchar NOT NULL, | mode character(1) DEFAULT 'D'::bpchar NOT NULL, |
| description text, | description text, |
| service_tag text, | service_tag text, |
| category text | category text |
| ); | ); |
| ALTER TABLE public.tgids OWNER TO postgres; | ALTER TABLE public.tgids OWNER TO postgres; |
| -- | -- |
| -- Name: recordings_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: | -- Name: recordings_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: |
| -- | -- |
| ALTER TABLE ONLY recordings | ALTER TABLE ONLY recordings |
| ADD CONSTRAINT recordings_pkey PRIMARY KEY (filename); | ADD CONSTRAINT recordings_pkey PRIMARY KEY (filename); |
| -- | -- |
| -- Name: tgids_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: | -- Name: tgids_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: |
| -- | -- |
| ALTER TABLE ONLY tgids | ALTER TABLE ONLY tgids |
| ADD CONSTRAINT tgids_pkey PRIMARY KEY (tgid); | ADD CONSTRAINT tgids_pkey PRIMARY KEY (tgid); |
| -- | -- |
| -- Name: public; Type: ACL; Schema: -; Owner: postgres | -- Name: public; Type: ACL; Schema: -; Owner: postgres |
| -- | -- |
| REVOKE ALL ON SCHEMA public FROM PUBLIC; | REVOKE ALL ON SCHEMA public FROM PUBLIC; |
| REVOKE ALL ON SCHEMA public FROM postgres; | REVOKE ALL ON SCHEMA public FROM postgres; |
| GRANT ALL ON SCHEMA public TO postgres; | GRANT ALL ON SCHEMA public TO postgres; |
| GRANT ALL ON SCHEMA public TO PUBLIC; | GRANT ALL ON SCHEMA public TO PUBLIC; |
| -- | -- |
| -- PostgreSQL database dump complete | -- PostgreSQL database dump complete |
| -- | -- |
| CREATE TABLE "compilation" ( | CREATE TABLE "compilations" ( |
| "filename" text NOT NULL, | "filename" text NOT NULL, |
| "files" text ARRAY NOT NULL, | "files" text ARRAY NOT NULL, |
| "datetime" timestamp NOT NULL | "datetime" timestamp NOT NULL |
| ); | ); |
| CREATE TABLE "trunk_log" ( | CREATE TABLE "trunk_logs" ( |
| "id" text NOT NULL, | "id" text NOT NULL, |
| "datetime" integer NOT NULL, | "datetime" integer NOT NULL, |
| "site" integer NOT NULL, | "site" integer NOT NULL, |
| "action" text NOT NULL, | "action" text NOT NULL, |
| "sourcetype" character(1) NOT NULL, | "sourcetype" character(1) NOT NULL, |
| "sourceid" smallint NOT NULL, | "sourceid" smallint NOT NULL, |
| "targettype" character(1) NOT NULL, | "targettype" character(1) NOT NULL, |
| "targetid" smallint NOT NULL, | "targetid" smallint NOT NULL, |
| "channel" smallint NOT NULL, | "channel" smallint NOT NULL, |
| "calltype" text NOT NULL | "calltype" text NOT NULL |
| ); | ); |
| <?php | \<?php |
| include('common.inc.php'); | include('common.inc.php'); |
| function processHourly($hourly) { | function processHourly($hourly) { |
| $filename = str_replace(' ',"_",$hourly['tgid']).'-'.str_replace(' 00:00:00+1','',$hourly['aday']).'-'.$hourly['ahour'].'.3gp'; | global $conn; |
| $hfilename = str_replace(' ','_',$hourly['tgid']) . '-' . str_replace(Array(' 00:00:00+10',' 00:00:00+11'), '', $hourly['aday']) . '-' . $hourly['ahour'] . '.3gp'; | |
| $hfilename = $hourly['tgid'] . '-' . str_replace(' 00:00:00+10', '', $hourly['aday']) . '-' . $hourly['ahour'] . '.3gp'; | |
| if (!file_exists("hourly/" . $hfilename)) { | if (!file_exists("hourly/" . $hfilename)) { |
| $filenames = explode(",", $hourly['filenames']); | $filenames = explode(",", $hourly['filenames']); |
| $cmd = "/usr/local/bin/ffmpeg -filter_complex concat=n=" . count($filenames) . ":v=0:a=1 -i data/" . implode(" -i data/", $filenames) . " -ar 8000 -ab 4.75k -ac 1 hourly/" . $hfilename . ' 2>&1'; | $cmd = "/usr/local/bin/ffmpeg"; |
| if (count($filenames) > 1) { | |
| $cmd .=" -filter_complex concat=n=" . count($filenames) . ":v=0:a=1"; // only concat when more than 1 file | |
| } | |
| $cmd .=" -i data/" . implode(" -i data/", $filenames) . " -ar 8000 -ab 4.75k -ac 1 hourly/" . $hfilename . ' 2>&1'; | |
| //print_r($hourly); | //print_r($hourly); |
| exec($cmd, $output, $returncode); | exec($cmd, $output, $returncode); |
| echo $cmd . "<br>\n"; | echo $cmd . "<br>\n"; |
| if ($returncode != 10) { | if ($returncode != 0) { |
| echo $returncode; | |
| print_r($output); | print_r($output); |
| //die(); | unlink("hourly/" . $hfilename); // delete incomplete file |
| // die(); | |
| } else { | } else { |
| $q = " insert into compilations (filename files datetime) ('" . $hfilename . "',{'" . implode("', '", $filenames) . "'},'" . strtottime($hourly['aday'] . ' +' . $hourly['ahour'] . " hours')") . "' "; | |
| $time = strtotime($hourly['aday'] . ' +' . $hourly['ahour'] . " hours"); | |
| $q = " insert into compilations (filename,files,datetime) VALUES ('" . $hfilename . "', ARRAY['" . implode("', '", $filenames) . "'], to_timestamp('" . $time . "') );"; | |
| $conn->query($q); | |
| //echo $q."\n"; | |
| foreach ($filenames as $filename) { | foreach ($filenames as $filename) { |
| $q = "UPDATE recordings SET archived = '$hfilename' WHERE filename = '$filename' "; | $q = "UPDATE recordings SET archived = '$hfilename' WHERE filename = '$filename' ;"; |
| $conn->query($q); | |
| //echo $q."\n"; | |
| } | } |
| //die(); | |
| } | } |
| } | } |
| } | } |
| /*$sth = $conn->prepare("select tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, | /*$sth = $conn->prepare("select tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, |
| count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames from recordings group by tgid, ahour, aday order by aday DESC, ahour, tgid;"); | count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames from recordings group by tgid, ahour, aday order by aday DESC, ahour, tgid;"); |
| $sth = $conn->prepare("SELECT tgid, EXTRACT(HOUR FROM call_timestamp) ahour, date_trunc('day', call_timestamp) aday, COUNT(filename), array_to_string(array_agg(filename ORDER BY call_timestamp), ',') filenames FROM recordings GROUP BY tgid, ahour, aday ORDER BY aday DESC, ahour, tgid;"); | |
| // TODO use tgid categories instead, tgid too specific | // TODO use tgid categories instead, tgid too specific |
| $sth->execute(); | $sth->execute(); |
| $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); | $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); |
| foreach ($hourlies as $hourly) { | foreach ($hourlies as $hourly) { |
| processHourly($hourly); | processHourly($hourly); |
| } | } |
| $sth = $conn->prepare("select 'hour' as tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, | $sth = $conn->prepare("select 'hour' as tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, |
| count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames from recordings group by ahour, aday order by aday DESC, ahour;"); | count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames from recordings group by ahour, aday order by aday DESC, ahour;"); |
| $sth->execute(); | $sth->execute(); |
| $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); | $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); |
| foreach ($hourlies as $hourly) { | foreach ($hourlies as $hourly) { |
| processHourly($hourly); | processHourly($hourly); |
| } | }*/ |
| $sth = $conn->prepare("select coalesce(category,'unknown') as tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, | $sth = $conn->prepare("select coalesce(category,'unknown') as tgid, extract(hour from call_timestamp) ahour, date_trunc('day', call_timestamp) aday, |
| count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames | count(filename), array_to_string(array_agg(filename order by call_timestamp), ',') filenames |
| from recordings inner join tgids on recordings.tgid = tgids.tgid group by category, ahour, aday order by aday DESC, ahour, category;"); | from recordings inner join tgids on recordings.tgid = tgids.tgid |
| group by category, ahour, aday | |
| having count(archived) != count(filename) | |
| order by aday DESC, ahour, category;"); | |
| $sth->execute(); | $sth->execute(); |
| $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); | $hourlies = $sth->fetchAll(PDO::FETCH_ASSOC); |
| foreach($hourlies as $hourly) { | foreach($hourlies as $hourly) { |
| processHourly($hourly); | processHourly($hourly); |
| } | } |
| """ Record a few seconds of audio and save to a WAVE file. | """ Record a few seconds of audio and save to a WAVE file. |
| Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593 | Based on http://stackoverflow.com/questions/892199/detect-record-audio-in-python/6743593#6743593 |
| """ | """ |
| import pyaudio | import pyaudio |
| import wave | import wave |
| import sys | import sys |
| from array import array | from array import array |
| from struct import unpack, pack | from struct import unpack, pack |
| from pydispatch import dispatcher | from pydispatch import dispatcher |
| THRESHOLD = 500 | THRESHOLD = 500 |
| CHUNK_SIZE = 1024 | CHUNK_SIZE = 1024 |
| FORMAT = pyaudio.paInt16 | FORMAT = pyaudio.paInt16 |
| RATE = 44100 | RATE = 44100 |
| if sys.platform.startswith('darwin'): | if sys.platform.startswith('darwin'): |
| CHANNELS = 2 | CHANNELS = 2 |
| elif sys.platform.startswith('win32'): | elif sys.platform.startswith('win32'): |
| CHANNELS = 1 | CHANNELS = 1 |
| MAX_SILENT = 80 | MAX_SILENT = 80 |
| def is_silent(L): | def is_silent(L): |
| "Returns `True` if below the 'silent' threshold" | "Returns `True` if below the 'silent' threshold" |
| "print max(L)" | "print max(L)" |
| "print max(L) < THRESHOLD" | "print max(L) < THRESHOLD" |
| return max(L) < THRESHOLD | return max(L) < THRESHOLD |
| def normalize(L): | def normalize(L): |
| "Average the volume out" | "Average the volume out" |
| MAXIMUM = 16384 | MAXIMUM = 16384 |
| times = float(MAXIMUM) / max(abs(i) for i in L) | times = float(MAXIMUM) / max(abs(i) for i in L) |
| LRtn = array('h') | LRtn = array('h') |
| for i in L: | for i in L: |
| LRtn.append(int(i * times)) | LRtn.append(int(i * times)) |
| return LRtn | return LRtn |
| def trim(L): | def trim(L): |
| "Trim the blank spots at the start and end" | "Trim the blank spots at the start and end" |
| def _trim(L): | def _trim(L): |
| snd_started = False | snd_started = False |
| LRtn = array('h') | LRtn = array('h') |
| for i in L: | for i in L: |
| if not snd_started and abs(i) > THRESHOLD: | if not snd_started and abs(i) > THRESHOLD: |
| snd_started = True | snd_started = True |
| LRtn.append(i) | LRtn.append(i) |
| elif snd_started: | elif snd_started: |
| LRtn.append(i) | LRtn.append(i) |
| return LRtn | return LRtn |
| # Trim to the left | # Trim to the left |
| L = _trim(L) | L = _trim(L) |
| # Trim to the right | # Trim to the right |
| L.reverse() | L.reverse() |
| L = _trim(L) | L = _trim(L) |
| L.reverse() | L.reverse() |
| return L | return L |
| def add_silence(L, seconds): | def add_silence(L, seconds): |
| "Add silence to the start and end of `L` of length `seconds` (float)" | "Add silence to the start and end of `L` of length `seconds` (float)" |
| LRtn = array('h', [0 for i in xrange(int(seconds * RATE))]) | LRtn = array('h', [0 for i in xrange(int(seconds * RATE))]) |
| LRtn.extend(L) | LRtn.extend(L) |
| LRtn.extend([0 for i in xrange(int(seconds * RATE))]) | LRtn.extend([0 for i in xrange(int(seconds * RATE))]) |
| return LRtn | return LRtn |
| def record(): | def record(): |
| """ | """ |
| Record a word or words from the microphone and | Record a word or words from the microphone and |
| return the data as an array of signed shorts. | return the data as an array of signed shorts. |
| Normalizes the audio, trims silence from the | Normalizes the audio, trims silence from the |
| start and end, and pads with 0.5 seconds of | start and end, and pads with 0.5 seconds of |
| blank sound to make sure VLC et al can play | blank sound to make sure VLC et al can play |
| it without getting chopped off. | it without getting chopped off. |
| """ | """ |
| p = pyaudio.PyAudio() | p = pyaudio.PyAudio() |
| stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, | stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, |
| input=True, | input=True, |
| frames_per_buffer=CHUNK_SIZE) | frames_per_buffer=CHUNK_SIZE) |
| num_silent = 0 | num_silent = 0 |
| snd_started = False | snd_started = False |
| LRtn = array('h') | LRtn = array('h') |
| while 1: | while 1: |
| try: | try: |
| data = stream.read(CHUNK_SIZE) | data = stream.read(CHUNK_SIZE) # Fewer than n bytes may be returned if the operating system call returns fewer than n bytes. |
| #if len(data) < 4096: | |
| #print "cropped packet detected! " + len(data) | |
| #data.extend('\x00' * CHUNK_SIZE - len(data)) | |
| #print "fixed to " + len(data) | |
| except IOError as ex: | except IOError as ex: |
| if ex[1] != pyaudio.paInputOverflowed: | if ex[1] != pyaudio.paInputOverflowed: |
| raise | raise |
| data = '\x00' * CHUNK_SIZE | data = '\x00' * CHUNK_SIZE |
| L = unpack('<' + ('h' * (len(data) / 2)), data) # little endian, signed short | L = unpack('<' + ('h' * (len(data) / 2)), data) # little endian, signed short |
| L = array('h', L) | L = array('h', L) |
| silent = is_silent(L) | silent = is_silent(L) |
| #print silent, num_silent, L[:10] | #print silent, num_silent, L[:10] |
| if silent and snd_started: | if silent and snd_started: |
| num_silent += 1 | num_silent += 1 |
| #print num_silent | #print num_silent |
| elif not silent and not snd_started: | elif not silent and not snd_started: |
| dispatcher.send(signal='SND_STARTED') | dispatcher.send(signal='SND_STARTED') |
| snd_started = True | snd_started = True |
| print snd_started | print snd_started |
| if snd_started and not silent: | if snd_started and not silent: |
| num_silent = 0 | num_silent = 0 |
| if snd_started: | if snd_started: |
| LRtn.extend(L) | LRtn.extend(L) |
| if snd_started and num_silent > MAX_SILENT: | if snd_started and num_silent > MAX_SILENT: |
| break | break |
| sample_width = p.get_sample_size(FORMAT) | sample_width = p.get_sample_size(FORMAT) |
| stream.stop_stream() | stream.stop_stream() |
| stream.close() | stream.close() |
| p.terminate() | p.terminate() |
| LRtn = normalize(LRtn) | LRtn = normalize(LRtn) |
| LRtn = trim(LRtn) | LRtn = trim(LRtn) |
| LRtn = add_silence(LRtn, 0.5) | LRtn = add_silence(LRtn, 0.5) |
| return sample_width, LRtn | return sample_width, LRtn |
| def record_to_file(path): | def record_to_file(path): |
| "Records from the microphone and outputs the resulting data to `path`" | "Records from the microphone and outputs the resulting data to `path`" |
| sample_width, data = record() | sample_width, data = record() |
| data = pack('<' + ('h' * len(data)), *data) | data = pack('<' + ('h' * len(data)), *data) |
| wf = wave.open(path, 'wb') | wf = wave.open(path, 'wb') |
| wf.setnchannels(CHANNELS) | wf.setnchannels(CHANNELS) |
| wf.setsampwidth(sample_width) | wf.setsampwidth(sample_width) |
| wf.setframerate(RATE) | wf.setframerate(RATE) |
| wf.writeframes(data) | wf.writeframes(data) |
| wf.close() | wf.close() |
| print("done - result written to " + path) | print("done - result written to " + path) |
| del data | del data |
| if __name__ == '__main__': | if __name__ == '__main__': |
| filename = 'demo.wav' | filename = 'demo.wav' |
| record_to_file(filename) | record_to_file(filename) |
| print("done - result written to " + filename) | print("done - result written to " + filename) |