From bd8e66519886e39bfa942841e4ab9838a1b01316 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Fri, 3 May 2019 22:59:24 +0200 Subject: [PATCH 01/73] add new webserver infrastructure --- htdocs/index.wrx | 54 ++++++++++++++++++++++----------------------- owrx/controllers.py | 43 ++++++++++++++++++++++++++++++++++++ owrx/http.py | 35 +++++++++++++++++++++++++++++ server.py | 6 +++++ 4 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 owrx/controllers.py create mode 100644 owrx/http.py create mode 100644 server.py diff --git a/htdocs/index.wrx b/htdocs/index.wrx index 5e00506..2cd64da 100644 --- a/htdocs/index.wrx +++ b/htdocs/index.wrx @@ -39,40 +39,40 @@ var mathbox_waterfall_history_length=%[MATHBOX_WATERFALL_THIST]; var mathbox_waterfall_colors=%[MATHBOX_WATERFALL_COLORS]; - - - - - - - + + + + + + +
- +
%[RX_PHOTO_TITLE]
%[RX_PHOTO_DESC]
- - - - + + + +
%[RX_TITLE]
%[RX_LOC] | Loc: %[RX_QRA], ASL: %[RX_ASL] m, [maps]
- - + +
    -

  • Status
    • -

  • Log
    • -

  • Receiver
    • +

  • Status
    • +

  • Log
    • +

  • Receiver
@@ -110,23 +110,23 @@
-
+
-
+
-
+
-
+
-
-
-
-
-
+
+
+
+
+
0 dB
@@ -172,7 +172,7 @@
- +

Start OpenWebRX
diff --git a/owrx/controllers.py b/owrx/controllers.py new file mode 100644 index 0000000..c7989c1 --- /dev/null +++ b/owrx/controllers.py @@ -0,0 +1,43 @@ +import mimetypes + +class Controller(object): + def __init__(self, handler, matches): + self.handler = handler + self.matches = matches + def send_response(self, content, code = 200, content_type = "text/html"): + self.handler.send_response(code) + if content_type is not None: + self.handler.send_header("Content-Type", content_type) + self.handler.end_headers() + if (type(content) == str): + content = content.encode() + self.handler.wfile.write(content) + def serve_file(self, file): + try: + f = open('htdocs/' + file, 'rb') + data = f.read() + f.close() + + (content_type, encoding) = mimetypes.MimeTypes().guess_type(file) + self.send_response(data, content_type = content_type) + except FileNotFoundError: + self.send_response("file not found", code = 404) + def render_template(self, template, **variables): + f = open('htdocs/' + template) + data = f.read() + f.close() + + self.send_response(data) + +class StatusController(Controller): + def handle_request(self): + self.send_response("you have reached the status page!") + +class IndexController(Controller): + def handle_request(self): + self.render_template("index.wrx") + +class AssetsController(Controller): + def handle_request(self): + filename = self.matches.group(1) + self.serve_file(filename) \ No newline at end of file diff --git a/owrx/http.py b/owrx/http.py new file mode 100644 index 0000000..b5ac0ae --- /dev/null +++ b/owrx/http.py @@ -0,0 +1,35 @@ +from owrx.controllers import StatusController, IndexController, AssetsController +from http.server import BaseHTTPRequestHandler +import re + +class RequestHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server): + self.router = Router() + super().__init__(request, client_address, server) + def do_GET(self): + self.router.route(self) + +class Router(object): + mappings = [ + {"route": "/", "controller": IndexController}, + {"route": "/status", "controller": StatusController}, + {"regex": "/static/(.+)", "controller": AssetsController} + ] + def find_controller(self, path): + for m in Router.mappings: + if "route" in m: + if m["route"] == path: + return (m["controller"], None) + if "regex" in m: + regex = re.compile(m["regex"]) + matches = regex.match(path) + if matches: + return (m["controller"], matches) + def route(self, handler): + res = self.find_controller(handler.path) + #print("path: {0}, controller: {1}, matches: {2}".format(handler.path, controller, matches)) + if res is not None: + (controller, matches) = res + controller(handler, matches).handle_request() + else: + handler.send_error(404, "Not Found", "The page you requested could not be found.") diff --git a/server.py b/server.py new file mode 100644 index 0000000..2b14a52 --- /dev/null +++ b/server.py @@ -0,0 +1,6 @@ +from http.server import HTTPServer +from owrx.http import RequestHandler + +server = HTTPServer(('0.0.0.0', 3000), RequestHandler) +server.serve_forever() + From 89690d214d8f2091142cd9f232db28b553d6f8b5 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sat, 4 May 2019 16:56:23 +0200 Subject: [PATCH 02/73] first work on the websocket connection --- htdocs/index.wrx | 11 ++------- htdocs/openwebrx.js | 57 ++++++++++++++++++++++++++++++++++++--------- owrx/config.py | 23 ++++++++++++++++++ owrx/controllers.py | 41 +++++++++++++++++++++++--------- owrx/http.py | 5 ++-- owrx/websocket.py | 47 +++++++++++++++++++++++++++++++++++++ server.py | 7 ++++++ 7 files changed, 158 insertions(+), 33 deletions(-) create mode 100644 owrx/config.py create mode 100644 owrx/websocket.py diff --git a/htdocs/index.wrx b/htdocs/index.wrx index 2cd64da..885e3ef 100644 --- a/htdocs/index.wrx +++ b/htdocs/index.wrx @@ -22,23 +22,16 @@ OpenWebRX | Open Source SDR Web App for Everyone! - --> diff --git a/htdocs/openwebrx.js b/htdocs/openwebrx.js index c7fd7c0..6efc93d 100644 --- a/htdocs/openwebrx.js +++ b/htdocs/openwebrx.js @@ -79,7 +79,8 @@ is_chrome = /Chrome/.test(navigator.userAgent); function init_rx_photo() { - e("webrx-top-photo-clip").style.maxHeight=rx_photo_height.toString()+"px"; + var clip = e("webrx-top-photo-clip"); + clip.style.maxHeight=clip.clientHeight+"px"; window.setTimeout(function() { animate(e("webrx-rx-photo-title"),"opacity","",1,0,1,500,30); },1000); window.setTimeout(function() { animate(e("webrx-rx-photo-desc"),"opacity","",1,0,1,500,30); },1500); window.setTimeout(function() { close_rx_photo() },2500); @@ -1145,6 +1146,46 @@ var COMPRESS_FFT_PAD_N=10; //should be the same as in csdr.c function on_ws_recv(evt) { + if (typeof evt.data == 'string') { + // text messages + if (evt.data.substr(0, 16) == "CLIENT DE SERVER") { + divlog("Server acknowledged WebSocket connection."); + } else { + try { + json = JSON.parse(evt.data) + switch (json.type) { + case "config": + config = json.value; + window.waterfall_colors = config.waterfall_colors; + window.waterfall_min_level_default = config.waterfall_min_level; + window.waterfall_max_level_default = config.waterfall_max_level; + window.waterfall_auto_level_margin = config.waterfall_auto_level_margin; + waterfallColorsDefault(); + + bandwidth = config.samp_rate; + center_freq = config.shown_center_freq; + fft_size = config.fft_size; + fft_fps = config.fft_fps; + audio_compression = config.audio_compression; + divlog( "Audio stream is "+ ((audio_compression=="adpcm")?"compressed":"uncompressed")+"." ) + fft_compression = config.fft_compression; + divlog( "FFT stream is "+ ((fft_compression=="adpcm")?"compressed":"uncompressed")+"." ) + max_clients_num = config.max_clients; + waterfall_init(); + audio_preinit(); + break; + default: + console.warn('received message of unknown type', json); + } + } catch (e) { + // don't lose exception + console.error(e) + } + } + } else if (evt.data instanceof ArrayBuffer) { + // binary messages + } + return if(!(evt.data instanceof ArrayBuffer)) { divlog("on_ws_recv(): Not ArrayBuffer received...",1); return; } // debug_ws_data_received+=evt.data.byteLength/1000; @@ -1152,8 +1193,6 @@ function on_ws_recv(evt) first3Chars=first4Chars.slice(0,3); if(first3Chars=="CLI") { - var stringData=arrayBufferToString(evt.data); - if(stringData.substring(0,16)=="CLIENT DE SERVER") divlog("Server acknowledged WebSocket connection."); } if(first3Chars=="AUD") @@ -1574,7 +1613,7 @@ function parsehash() if(harr[0]=="mute") toggleMute(); else if(harr[0]=="mod") starting_mod = harr[1]; else if(harr[0]=="sql") - { + { config e("openwebrx-panel-squelch").value=harr[1]; updateSquelch(); } @@ -1692,14 +1731,10 @@ String.prototype.startswith=function(str){ return this.indexOf(str) == 0; }; //h function open_websocket() { - //if(ws_url.startswith("ws://localhost:")&&window.location.hostname!="127.0.0.1"&&window.location.hostname!="localhost") - //{ - //divlog("Server administrator should set server_hostname correctly, because it is left as \"localhost\". Now guessing hostname from page URL.",1); - ws_url="ws://"+(window.location.origin.split("://")[1])+"/ws/"; //guess automatically -> now default behaviour - //} + ws_url="ws://"+(window.location.origin.split("://")[1])+"/ws/"; //guess automatically -> now default behaviour if (!("WebSocket" in window)) divlog("Your browser does not support WebSocket, which is required for WebRX to run. Please upgrade to a HTML5 compatible browser."); - ws = new WebSocket(ws_url+client_id); + ws = new WebSocket(ws_url); ws.onopen = on_ws_opened; ws.onmessage = on_ws_recv; ws.onclose = on_ws_closed; @@ -2196,7 +2231,7 @@ function openwebrx_init() //Synchronise volume with slider updateVolume(); - waterfallColorsDefault(); + } function iosPlayButtonClick() diff --git a/owrx/config.py b/owrx/config.py new file mode 100644 index 0000000..7e4e7e5 --- /dev/null +++ b/owrx/config.py @@ -0,0 +1,23 @@ +class Property(object): + def __init__(self, value = None): + self.value = value + def getValue(self): + return self.value + def setValue(self, value): + self.value = value + +class PropertyManager(object): + sharedInstance = None + @staticmethod + def getSharedInstance(): + if PropertyManager.sharedInstance is None: + PropertyManager.sharedInstance = PropertyManager() + return PropertyManager.sharedInstance + + def __init__(self): + self.properties = {} + + def getProperty(self, name): + if not name in self.properties: + self.properties[name] = Property() + return self.properties[name] diff --git a/owrx/controllers.py b/owrx/controllers.py index c7989c1..12a4aa2 100644 --- a/owrx/controllers.py +++ b/owrx/controllers.py @@ -1,4 +1,6 @@ import mimetypes +from owrx.websocket import WebSocketConnection +from owrx.config import PropertyManager class Controller(object): def __init__(self, handler, matches): @@ -12,16 +14,6 @@ class Controller(object): if (type(content) == str): content = content.encode() self.handler.wfile.write(content) - def serve_file(self, file): - try: - f = open('htdocs/' + file, 'rb') - data = f.read() - f.close() - - (content_type, encoding) = mimetypes.MimeTypes().guess_type(file) - self.send_response(data, content_type = content_type) - except FileNotFoundError: - self.send_response("file not found", code = 404) def render_template(self, template, **variables): f = open('htdocs/' + template) data = f.read() @@ -38,6 +30,33 @@ class IndexController(Controller): self.render_template("index.wrx") class AssetsController(Controller): + def serve_file(self, file): + try: + f = open('htdocs/' + file, 'rb') + data = f.read() + f.close() + + (content_type, encoding) = mimetypes.MimeTypes().guess_type(file) + self.send_response(data, content_type = content_type) + except FileNotFoundError: + self.send_response("file not found", code = 404) def handle_request(self): filename = self.matches.group(1) - self.serve_file(filename) \ No newline at end of file + self.serve_file(filename) + + +class WebSocketController(Controller): + def handle_request(self): + conn = WebSocketConnection(self.handler) + conn.send("CLIENT DE SERVER openwebrx.py") + + config = {} + pm = PropertyManager.getSharedInstance() + + for key in ["waterfall_colors", "waterfall_min_level", "waterfall_max_level", "waterfall_auto_level_margin", + "shown_center_freq", "samp_rate", "fft_size", "fft_fps", "audio_compression", "fft_compression", + "max_clients"]: + + config[key] = pm.getProperty(key).getValue() + + conn.send({"type":"config","value":config}) diff --git a/owrx/http.py b/owrx/http.py index b5ac0ae..ab4399b 100644 --- a/owrx/http.py +++ b/owrx/http.py @@ -1,4 +1,4 @@ -from owrx.controllers import StatusController, IndexController, AssetsController +from owrx.controllers import StatusController, IndexController, AssetsController, WebSocketController from http.server import BaseHTTPRequestHandler import re @@ -13,7 +13,8 @@ class Router(object): mappings = [ {"route": "/", "controller": IndexController}, {"route": "/status", "controller": StatusController}, - {"regex": "/static/(.+)", "controller": AssetsController} + {"regex": "/static/(.+)", "controller": AssetsController}, + {"route": "/ws/", "controller": WebSocketController} ] def find_controller(self, path): for m in Router.mappings: diff --git a/owrx/websocket.py b/owrx/websocket.py new file mode 100644 index 0000000..8bd305e --- /dev/null +++ b/owrx/websocket.py @@ -0,0 +1,47 @@ +import base64 +import hashlib +import json + +class WebSocketConnection(object): + def __init__(self, handler): + self.handler = handler + my_headers = self.handler.headers.items() + my_header_keys = list(map(lambda x:x[0],my_headers)) + h_key_exists = lambda x:my_header_keys.count(x) + h_value = lambda x:my_headers[my_header_keys.index(x)][1] + if (not h_key_exists("Upgrade")) or not (h_value("Upgrade")=="websocket") or (not h_key_exists("Sec-WebSocket-Key")): + raise WebSocketException + ws_key = h_value("Sec-WebSocket-Key") + shakey = hashlib.sha1() + shakey.update("{ws_key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11".format(ws_key = ws_key).encode()) + ws_key_toreturn = base64.b64encode(shakey.digest()) + self.handler.wfile.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {0}\r\nCQ-CQ-de: HA5KFU\r\n\r\n".format(ws_key_toreturn.decode()).encode()) + + def get_header(self, size, opcode): + ws_first_byte = 0b10000000 | (opcode & 0x0F) + if(size>125): + return bytes([ws_first_byte, 126, (size>>8) & 0xff, size & 0xff]) + else: + # 256 bytes binary message in a single unmasked frame + return bytes([ws_first_byte, size]) + + def send(self, data): + # convenience + if (type(data) == dict): + data = json.dumps(data) + + # string-type messages are sent as text frames + if (type(data) == str): + header = self.get_header(len(data), 1) + self.handler.wfile.write(header) + self.handler.wfile.write(data.encode('utf-8')) + self.handler.wfile.flush() + # anything else as binary + else: + header = self.get_header(len(data), 2) + self.handler.wfile.write(header) + self.handler.wfile.write(data.encode()) + self.handler.wfile.flush() + +class WebSocketException(Exception): + pass diff --git a/server.py b/server.py index 2b14a52..32850a5 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,12 @@ from http.server import HTTPServer from owrx.http import RequestHandler +from owrx.config import PropertyManager + +cfg=__import__("config_webrx") +pm = PropertyManager.getSharedInstance() +for name, value in cfg.__dict__.items(): + if (name.startswith("__")): continue + pm.getProperty(name).setValue(value) server = HTTPServer(('0.0.0.0', 3000), RequestHandler) server.serve_forever() From 1f909080db4aed248299f37e769cb189023c3493 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sat, 4 May 2019 20:26:11 +0200 Subject: [PATCH 03/73] we got fft --- config_webrx.py | 2 +- csdr.py | 14 +++--- htdocs/openwebrx.js | 51 ++++++++++++++-------- owrx/config.py | 3 ++ owrx/controllers.py | 74 ++++++++++++++++++++++++++------ owrx/source.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ owrx/websocket.py | 32 +++++++++++++- server.py | 24 ++++++++--- 8 files changed, 254 insertions(+), 47 deletions(-) create mode 100644 owrx/source.py diff --git a/config_webrx.py b/config_webrx.py index 34e480c..a4f63ef 100644 --- a/config_webrx.py +++ b/config_webrx.py @@ -175,7 +175,7 @@ iq_server_port = 4951 #TCP port for ncat to listen on. It will send I/Q data ove #A guide is available to help you set these values: https://github.com/simonyiszk/openwebrx/wiki/Calibrating-waterfall-display-levels ### default theme by teejez: -waterfall_colors = "[0x000000ff,0x0000ffff,0x00ffffff,0x00ff00ff,0xffff00ff,0xff0000ff,0xff00ffff,0xffffffff]" +waterfall_colors = [0x000000ff,0x0000ffff,0x00ffffff,0x00ff00ff,0xffff00ff,0xff0000ff,0xff00ffff,0xffffffff] waterfall_min_level = -88 #in dB waterfall_max_level = -20 waterfall_auto_level_margin = (5, 40) diff --git a/csdr.py b/csdr.py index a2fb490..a7f1f89 100755 --- a/csdr.py +++ b/csdr.py @@ -129,7 +129,7 @@ class dsp: def start_secondary_demodulator(self): if(not self.secondary_demodulator): return - print "[openwebrx] starting secondary demodulator from IF input sampled at %d"%self.if_samp_rate() + print("[openwebrx] starting secondary demodulator from IF input sampled at %d"%self.if_samp_rate()) secondary_command_fft=self.secondary_chain("fft") secondary_command_demod=self.secondary_chain(self.secondary_demodulator) self.try_create_pipes(self.secondary_pipe_names, secondary_command_demod + secondary_command_fft) @@ -150,16 +150,16 @@ class dsp: if_samp_rate=self.if_samp_rate() ) - print "[openwebrx-dsp-plugin:csdr] secondary command (fft) =", secondary_command_fft - print "[openwebrx-dsp-plugin:csdr] secondary command (demod) =", secondary_command_demod + print("[openwebrx-dsp-plugin:csdr] secondary command (fft) =", secondary_command_fft) + print("[openwebrx-dsp-plugin:csdr] secondary command (demod) =", secondary_command_demod) #code.interact(local=locals()) my_env=os.environ.copy() #if self.csdr_dynamic_bufsize: my_env["CSDR_DYNAMIC_BUFSIZE_ON"]="1"; if self.csdr_print_bufsizes: my_env["CSDR_PRINT_BUFSIZES"]="1"; self.secondary_process_fft = subprocess.Popen(secondary_command_fft, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setpgrp, env=my_env) - print "[openwebrx-dsp-plugin:csdr] Popen on secondary command (fft)" + print("[openwebrx-dsp-plugin:csdr] Popen on secondary command (fft)") self.secondary_process_demod = subprocess.Popen(secondary_command_demod, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setpgrp, env=my_env) #TODO digimodes - print "[openwebrx-dsp-plugin:csdr] Popen on secondary command (demod)" #TODO digimodes + print("[openwebrx-dsp-plugin:csdr] Popen on secondary command (demod)") #TODO digimodes self.secondary_processes_running = True #open control pipes for csdr and send initialization data @@ -313,7 +313,7 @@ class dsp: pipe_path = getattr(self,pipe_name,None) if pipe_path: try: os.unlink(pipe_path) - except Exception as e: print "[openwebrx-dsp-plugin:csdr] try_delete_pipes() ::", e + except Exception as e: print("[openwebrx-dsp-plugin:csdr] try_delete_pipes() ::", e) def set_pipe_nonblocking(self, pipe): flags = fcntl.fcntl(pipe, fcntl.F_GETFL) @@ -354,7 +354,7 @@ class dsp: flowcontrol=int(self.samp_rate*2), start_bufsize=self.base_bufsize*self.decimation, nc_port=self.nc_port, \ squelch_pipe=self.squelch_pipe, smeter_pipe=self.smeter_pipe, iqtee_pipe=self.iqtee_pipe, iqtee2_pipe=self.iqtee2_pipe ) - print "[openwebrx-dsp-plugin:csdr] Command =",command + print("[openwebrx-dsp-plugin:csdr] Command =",command) #code.interact(local=locals()) my_env=os.environ.copy() if self.csdr_dynamic_bufsize: my_env["CSDR_DYNAMIC_BUFSIZE_ON"]="1"; diff --git a/htdocs/openwebrx.js b/htdocs/openwebrx.js index 6efc93d..de12eea 100644 --- a/htdocs/openwebrx.js +++ b/htdocs/openwebrx.js @@ -1184,6 +1184,25 @@ function on_ws_recv(evt) } } else if (evt.data instanceof ArrayBuffer) { // binary messages + type = new Uint8Array(evt.data, 0, 1)[0] + data = evt.data.slice(1) + + switch (type) { + case 1: + if (fft_compression=="none") { + waterfall_add_queue(new Float32Array(data)); + } else if (fft_compression == "adpcm") { + fft_codec.reset(); + + var waterfall_i16=fft_codec.decode(new Uint8Array(data)); + var waterfall_f32=new Float32Array(waterfall_i16.length-COMPRESS_FFT_PAD_N); + for(var i=0;i>>0)>>((3-i)*8))&0xff; }*/ - if(mathbox_mode==MATHBOX_MODES.WATERFALL) - { + if (mathbox_mode==MATHBOX_MODES.WATERFALL) { //Handle mathbox for(var i=0;i>>0)>>((3-i)*8))&0xff; - } + } else { + //Add line to waterfall image + oneline_image = canvas_context.createImageData(w,1); + for (x=0;x>>0)>>((3-i)*8))&0xff; + } - //Draw image - canvas_context.putImageData(oneline_image, 0, canvas_actual_line--); - shift_canvases(); - if(canvas_actual_line<0) add_canvas(); + //Draw image + canvas_context.putImageData(oneline_image, 0, canvas_actual_line--); + shift_canvases(); + if(canvas_actual_line<0) add_canvas(); } diff --git a/owrx/config.py b/owrx/config.py index 7e4e7e5..9866c98 100644 --- a/owrx/config.py +++ b/owrx/config.py @@ -21,3 +21,6 @@ class PropertyManager(object): if not name in self.properties: self.properties[name] = Property() return self.properties[name] + + def getPropertyValue(self, name): + return self.getProperty(name).getValue() diff --git a/owrx/controllers.py b/owrx/controllers.py index 12a4aa2..f1c6ccd 100644 --- a/owrx/controllers.py +++ b/owrx/controllers.py @@ -1,6 +1,9 @@ import mimetypes from owrx.websocket import WebSocketConnection from owrx.config import PropertyManager +from owrx.source import SpectrumThread +import csdr +import json class Controller(object): def __init__(self, handler, matches): @@ -44,19 +47,66 @@ class AssetsController(Controller): filename = self.matches.group(1) self.serve_file(filename) +class SpectrumForwarder(object): + def __init__(self, conn): + self.conn = conn + def write_spectrum_data(self, data): + self.conn.send(bytes([0x01]) + data) + +class WebSocketMessageHandler(object): + def __init__(self): + self.forwarder = None + + def handleTextMessage(self, conn, message): + if (message[:16] == "SERVER DE CLIENT"): + config = {} + pm = PropertyManager.getSharedInstance() + + for key in ["waterfall_colors", "waterfall_min_level", "waterfall_max_level", "waterfall_auto_level_margin", + "shown_center_freq", "samp_rate", "fft_size", "fft_fps", "audio_compression", "fft_compression", + "max_clients"]: + + config[key] = pm.getProperty(key).getValue() + + conn.send({"type":"config","value":config}) + print("client connection intitialized") + + dsp = self.dsp = csdr.dsp() + dsp_initialized=False + dsp.set_audio_compression(pm.getPropertyValue("audio_compression")) + dsp.set_fft_compression(pm.getPropertyValue("fft_compression")) #used by secondary chains + dsp.set_format_conversion(pm.getPropertyValue("format_conversion")) + dsp.set_offset_freq(0) + dsp.set_bpf(-4000,4000) + dsp.set_secondary_fft_size(pm.getPropertyValue("digimodes_fft_size")) + dsp.nc_port=pm.getPropertyValue("iq_server_port") + dsp.csdr_dynamic_bufsize = pm.getPropertyValue("csdr_dynamic_bufsize") + dsp.csdr_print_bufsizes = pm.getPropertyValue("csdr_print_bufsizes") + dsp.csdr_through = pm.getPropertyValue("csdr_through") + do_secondary_demod=False + + self.forwarder = SpectrumForwarder(conn) + SpectrumThread.getSharedInstance().add_client(self.forwarder) + + else: + try: + message = json.loads(message) + if message["type"] == "start": + self.dsp.set_samp_rate(message["params"]["output_rate"]) + self.dsp.start() + except json.JSONDecodeError: + print("message is not json: {0}".format(message)) + + def handleBinaryMessage(self, conn, data): + print("unsupported binary message, discarding") + + def handleClose(self, conn): + if self.forwarder: + SpectrumThread.getSharedInstance().remove_client(self.forwarder) class WebSocketController(Controller): def handle_request(self): - conn = WebSocketConnection(self.handler) + conn = WebSocketConnection(self.handler, WebSocketMessageHandler()) conn.send("CLIENT DE SERVER openwebrx.py") - - config = {} - pm = PropertyManager.getSharedInstance() - - for key in ["waterfall_colors", "waterfall_min_level", "waterfall_max_level", "waterfall_auto_level_margin", - "shown_center_freq", "samp_rate", "fft_size", "fft_fps", "audio_compression", "fft_compression", - "max_clients"]: - - config[key] = pm.getProperty(key).getValue() - - conn.send({"type":"config","value":config}) + # enter read loop + conn.read_loop() diff --git a/owrx/source.py b/owrx/source.py new file mode 100644 index 0000000..0f18014 --- /dev/null +++ b/owrx/source.py @@ -0,0 +1,101 @@ +import subprocess +from owrx.config import PropertyManager +import threading +import csdr +import time + +class RtlNmuxSource(object): + def __init__(self): + pm = PropertyManager.getSharedInstance() + + nmux_bufcnt = nmux_bufsize = 0 + while nmux_bufsize < pm.getPropertyValue("samp_rate")/4: nmux_bufsize += 4096 + while nmux_bufsize * nmux_bufcnt < pm.getPropertyValue("nmux_memory") * 1e6: nmux_bufcnt += 1 + if nmux_bufcnt == 0 or nmux_bufsize == 0: + print("[openwebrx-main] Error: nmux_bufsize or nmux_bufcnt is zero. These depend on nmux_memory and samp_rate options in config_webrx.py") + return + print("[openwebrx-main] nmux_bufsize = %d, nmux_bufcnt = %d" % (nmux_bufsize, nmux_bufcnt)) + cmd = pm.getPropertyValue("start_rtl_command") + "| nmux --bufsize %d --bufcnt %d --port %d --address 127.0.0.1" % (nmux_bufsize, nmux_bufcnt, pm.getPropertyValue("iq_server_port")) + subprocess.Popen(cmd, shell=True) + print("[openwebrx-main] Started rtl source: " + cmd) + +class SpectrumThread(threading.Thread): + sharedInstance = None + @staticmethod + def getSharedInstance(): + if SpectrumThread.sharedInstance is None: + SpectrumThread.sharedInstance = SpectrumThread() + SpectrumThread.sharedInstance.start() + return SpectrumThread.sharedInstance + + def __init__(self): + self.clients = [] + self.doRun = True + super().__init__() + + def run(self): + pm = PropertyManager.getSharedInstance() + + samp_rate = pm.getPropertyValue("samp_rate") + fft_size = pm.getPropertyValue("fft_size") + fft_fps = pm.getPropertyValue("fft_fps") + fft_voverlap_factor = pm.getPropertyValue("fft_voverlap_factor") + fft_compression = pm.getPropertyValue("fft_compression") + format_conversion = pm.getPropertyValue("format_conversion") + + spectrum_dsp=dsp=csdr.dsp() + dsp.nc_port = pm.getPropertyValue("iq_server_port") + dsp.set_demodulator("fft") + dsp.set_samp_rate(samp_rate) + dsp.set_fft_size(fft_size) + dsp.set_fft_fps(fft_fps) + dsp.set_fft_averages(int(round(1.0 * samp_rate / fft_size / fft_fps / (1.0 - fft_voverlap_factor))) if fft_voverlap_factor>0 else 0) + dsp.set_fft_compression(fft_compression) + dsp.set_format_conversion(format_conversion) + dsp.csdr_dynamic_bufsize = pm.getPropertyValue("csdr_dynamic_bufsize") + dsp.csdr_print_bufsizes = pm.getPropertyValue("csdr_print_bufsizes") + dsp.csdr_through = pm.getPropertyValue("csdr_through") + sleep_sec=0.87/fft_fps + print("[openwebrx-spectrum] Spectrum thread initialized successfully.") + dsp.start() + if pm.getPropertyValue("csdr_dynamic_bufsize"): + dsp.read(8) #dummy read to skip bufsize & preamble + print("[openwebrx-spectrum] Note: CSDR_DYNAMIC_BUFSIZE_ON = 1") + print("[openwebrx-spectrum] Spectrum thread started.") + bytes_to_read=int(dsp.get_fft_bytes_to_read()) + spectrum_thread_counter=0 + while self.doRun: + data=dsp.read(bytes_to_read) + #print("gotcha",len(data),"bytes of spectrum data via spectrum_thread_function()") + if spectrum_thread_counter >= fft_fps: + spectrum_thread_counter=0 + else: spectrum_thread_counter+=1 + for c in self.clients: + c.write_spectrum_data(data) + ''' + correction=0 + for i in range(0,len(clients)): + i-=correction + if (clients[i].ws_started): + if clients[i].spectrum_queue.full(): + print "[openwebrx-spectrum] client spectrum queue full, closing it." + close_client(i, False) + correction+=1 + else: + clients[i].spectrum_queue.put([data]) # add new string by "reference" to all clients + ''' + + print("spectrum thread shut down") + + def add_client(self, c): + self.clients.append(c) + + def remove_client(self, c): + self.clients.remove(c) + if not self.clients: + self.shutdown() + + def shutdown(self): + print("shutting down spectrum thread") + SpectrumThread.sharedInstance = None + self.doRun = False \ No newline at end of file diff --git a/owrx/websocket.py b/owrx/websocket.py index 8bd305e..7c96e34 100644 --- a/owrx/websocket.py +++ b/owrx/websocket.py @@ -3,8 +3,9 @@ import hashlib import json class WebSocketConnection(object): - def __init__(self, handler): + def __init__(self, handler, messageHandler): self.handler = handler + self.messageHandler = messageHandler my_headers = self.handler.headers.items() my_header_keys = list(map(lambda x:x[0],my_headers)) h_key_exists = lambda x:my_header_keys.count(x) @@ -40,8 +41,35 @@ class WebSocketConnection(object): else: header = self.get_header(len(data), 2) self.handler.wfile.write(header) - self.handler.wfile.write(data.encode()) + self.handler.wfile.write(data) self.handler.wfile.flush() + def read_loop(self): + open = True + while (open): + header = self.handler.rfile.read(2) + opcode = header[0] & 0x0F + length = header[1] & 0x7F + mask = (header[1] & 0x80) >> 7 + if (length == 126): + header = self.handler.rfile.read(2) + length = (header[0] << 8) + header[1] + if (mask): + masking_key = self.handler.rfile.read(4) + data = self.handler.rfile.read(length) + print("opcode: {0}, length: {1}, mask: {2}".format(opcode, length, mask)) + if (mask): + data = bytes([b ^ masking_key[index % 4] for (index, b) in enumerate(data)]) + if (opcode == 1): + message = data.decode('utf-8') + self.messageHandler.handleTextMessage(self, message) + elif (opcode == 2): + self.messageHandler.handleBinaryMessage(self, data) + elif (opcode == 8): + open = False + self.messageHandler.handleClose(self) + else: + print("unsupported opcode: {0}".format(opcode)) + class WebSocketException(Exception): pass diff --git a/server.py b/server.py index 32850a5..c267f9e 100644 --- a/server.py +++ b/server.py @@ -1,13 +1,23 @@ from http.server import HTTPServer from owrx.http import RequestHandler from owrx.config import PropertyManager +from owrx.source import RtlNmuxSource, SpectrumThread +from socketserver import ThreadingMixIn -cfg=__import__("config_webrx") -pm = PropertyManager.getSharedInstance() -for name, value in cfg.__dict__.items(): - if (name.startswith("__")): continue - pm.getProperty(name).setValue(value) +class ThreadedHttpServer(ThreadingMixIn, HTTPServer): + pass -server = HTTPServer(('0.0.0.0', 3000), RequestHandler) -server.serve_forever() +def main(): + cfg=__import__("config_webrx") + pm = PropertyManager.getSharedInstance() + for name, value in cfg.__dict__.items(): + if (name.startswith("__")): continue + pm.getProperty(name).setValue(value) + RtlNmuxSource() + + server = ThreadedHttpServer(('0.0.0.0', 3000), RequestHandler) + server.serve_forever() + +if __name__=="__main__": + main() From 6ec21e6716d9a76127aaa9d56570e3349acc4b98 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sat, 4 May 2019 20:40:13 +0200 Subject: [PATCH 04/73] send missing parameters for audio client startup --- htdocs/index.wrx | 2 -- htdocs/openwebrx.js | 2 ++ owrx/controllers.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/htdocs/index.wrx b/htdocs/index.wrx index 885e3ef..f091933 100644 --- a/htdocs/index.wrx +++ b/htdocs/index.wrx @@ -25,8 +25,6 @@ From 210fe5352fa0afb41512d72c0e50746640490522 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 14:35:25 +0200 Subject: [PATCH 68/73] refactor the sdr.hu updater into the new server, too --- openwebrx.py | 12 ++++++------ owrx/config.py | 15 +++++++++++++++ owrx/sdrhu.py | 36 ++++++++++++++++++++++++++++++++++++ sdrhu.py | 32 +++++++------------------------- 4 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 owrx/sdrhu.py diff --git a/openwebrx.py b/openwebrx.py index 07df451..b89d7e5 100644 --- a/openwebrx.py +++ b/openwebrx.py @@ -3,6 +3,7 @@ from owrx.http import RequestHandler from owrx.config import PropertyManager, FeatureDetector from owrx.source import SdrService from socketserver import ThreadingMixIn +from owrx.sdrhu import SdrHuUpdater import logging logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -22,12 +23,7 @@ Author contact info: Andras Retzler, HA7ILM """) - cfg = __import__("config_webrx") - pm = PropertyManager.getSharedInstance() - for name, value in cfg.__dict__.items(): - if name.startswith("__"): - continue - pm[name] = value + pm = PropertyManager.getSharedInstance().loadConfig("config_webrx") featureDetector = FeatureDetector() if not featureDetector.is_available("core"): @@ -39,6 +35,10 @@ Author contact info: Andras Retzler, HA7ILM # Get error messages about unknown / unavailable features as soon as possible SdrService.loadProps() + if "sdrhu_key" in pm and pm["sdrhu_public_listing"]: + updater = SdrHuUpdater() + updater.start() + server = ThreadedHttpServer(('0.0.0.0', pm.getPropertyValue("web_port")), RequestHandler) server.serve_forever() diff --git a/owrx/config.py b/owrx/config.py index cc84c26..a117228 100644 --- a/owrx/config.py +++ b/owrx/config.py @@ -53,6 +53,9 @@ class PropertyManager(object): prop.wire(fireCallbacks) return self + def __contains__(self, name): + return self.hasProperty(name) + def __getitem__(self, name): return self.getPropertyValue(name) @@ -61,6 +64,9 @@ class PropertyManager(object): self.add(name, Property()) self.getProperty(name).setValue(value) + def __dict__(self): + return {k:v.getValue() for k, v in self.properties.items()} + def hasProperty(self, name): return name in self.properties @@ -86,6 +92,15 @@ class PropertyManager(object): p.setValue(other_pm[key]) return self + def loadConfig(self, filename): + cfg = __import__(filename) + for name, value in cfg.__dict__.items(): + if name.startswith("__"): + continue + self[name] = value + return self + + class UnknownFeatureException(Exception): pass diff --git a/owrx/sdrhu.py b/owrx/sdrhu.py new file mode 100644 index 0000000..b2c6b0f --- /dev/null +++ b/owrx/sdrhu.py @@ -0,0 +1,36 @@ +import threading +import subprocess +import time +from owrx.config import PropertyManager + +import logging +logger = logging.getLogger(__name__) + + +class SdrHuUpdater(threading.Thread): + def __init__(self): + self.doRun = True + super().__init__() + + def update(self): + pm = PropertyManager.getSharedInstance() + cmd = "wget --timeout=15 -4qO- https://sdr.hu/update --post-data \"url=http://{server_hostname}:{web_port}&apikey={sdrhu_key}\" 2>&1".format(**pm.__dict__()) + logger.debug(cmd) + returned=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate() + returned=returned[0].decode('utf-8') + if "UPDATE:" in returned: + retrytime_mins = 20 + value=returned.split("UPDATE:")[1].split("\n",1)[0] + if value.startswith("SUCCESS"): + logger.info("Update succeeded!") + else: + logger.warning("Update failed, your receiver cannot be listed on sdr.hu! Reason: %s", value) + else: + retrytime_mins = 2 + logger.warning("wget failed while updating, your receiver cannot be listed on sdr.hu!") + return retrytime_mins + + def run(self): + while self.doRun: + retrytime_mins = self.update() + time.sleep(60*retrytime_mins) diff --git a/sdrhu.py b/sdrhu.py index d06ae05..3060789 100755 --- a/sdrhu.py +++ b/sdrhu.py @@ -20,31 +20,13 @@ """ -import config_webrx as cfg, time, subprocess - -def run(continuously=True): - if not cfg.sdrhu_key: return - firsttime="(Your receiver is soon getting listed on sdr.hu!)" - while True: - cmd = "wget --timeout=15 -4qO- https://sdr.hu/update --post-data \"url=http://"+cfg.server_hostname+":"+str(cfg.web_port)+"&apikey="+cfg.sdrhu_key+"\" 2>&1" - print "[openwebrx-sdrhu]", cmd - returned=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate() - returned=returned[0] - #print returned - if "UPDATE:" in returned: - retrytime_mins = 20 - value=returned.split("UPDATE:")[1].split("\n",1)[0] - if value.startswith("SUCCESS"): - print "[openwebrx-sdrhu] Update succeeded! "+firsttime - firsttime="" - else: - print "[openwebrx-sdrhu] Update failed, your receiver cannot be listed on sdr.hu! Reason:", value - else: - retrytime_mins = 2 - print "[openwebrx-sdrhu] wget failed while updating, your receiver cannot be listed on sdr.hu!" - if not continuously: break - time.sleep(60*retrytime_mins) +from owrx.sdrhu import SdrHuUpdater +from owrx.config import PropertyManager if __name__=="__main__": - run(False) + pm = PropertyManager.getSharedInstance().loadConfig("config_webrx") + + if not "sdrhu_key" in pm: + exit(1) + SdrHuUpdater().update() From da37d03104a6ebf0c6cdd87fbdcd85d9a5fe96ca Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 15:56:18 +0200 Subject: [PATCH 69/73] refactor into more reasonable namespaces --- openwebrx.py | 3 +- owrx/config.py | 67 +--------------- owrx/connection.py | 180 +++++++++++++++++++++++++++++++++++++++++++ owrx/controllers.py | 182 +------------------------------------------- owrx/feature.py | 65 ++++++++++++++++ owrx/source.py | 3 +- 6 files changed, 254 insertions(+), 246 deletions(-) create mode 100644 owrx/connection.py create mode 100644 owrx/feature.py diff --git a/openwebrx.py b/openwebrx.py index b89d7e5..e41d6c7 100644 --- a/openwebrx.py +++ b/openwebrx.py @@ -1,6 +1,7 @@ from http.server import HTTPServer from owrx.http import RequestHandler -from owrx.config import PropertyManager, FeatureDetector +from owrx.config import PropertyManager +from owrx.feature import FeatureDetector from owrx.source import SdrService from socketserver import ThreadingMixIn from owrx.sdrhu import SdrHuUpdater diff --git a/owrx/config.py b/owrx/config.py index a117228..8fb6513 100644 --- a/owrx/config.py +++ b/owrx/config.py @@ -1,8 +1,7 @@ -import os - import logging logger = logging.getLogger(__name__) + class Property(object): def __init__(self, value = None): self.value = value @@ -99,67 +98,3 @@ class PropertyManager(object): continue self[name] = value return self - - -class UnknownFeatureException(Exception): - pass - -class RequirementMissingException(Exception): - pass - -class FeatureDetector(object): - features = { - "core": [ "csdr", "nmux" ], - "rtl_sdr": [ "rtl_sdr" ], - "sdrplay": [ "rx_tools" ], - "hackrf": [ "hackrf_transfer" ] - } - - def is_available(self, feature): - return self.has_requirements(self.get_requirements(feature)) - - def get_requirements(self, feature): - try: - return FeatureDetector.features[feature] - except KeyError: - raise UnknownFeatureException("Feature \"{0}\" is not known.".format(feature)) - - def has_requirements(self, requirements): - passed = True - for requirement in requirements: - methodname = "has_" + requirement - if hasattr(self, methodname) and callable(getattr(self, methodname)): - passed = passed and getattr(self, methodname)() - else: - logger.error("detection of requirement {0} not implement. please fix in code!".format(requirement)) - return passed - - def has_csdr(self): - return os.system("csdr 2> /dev/null") != 32512 - - def has_nmux(self): - return os.system("nmux --help 2> /dev/null") != 32512 - - def has_rtl_sdr(self): - return os.system("rtl_sdr --help 2> /dev/null") != 32512 - - def has_rx_tools(self): - return os.system("rx_sdr --help 2> /dev/null") != 32512 - - """ - To use a HackRF, compile the HackRF host tools from its "stdout" branch: - git clone https://github.com/mossmann/hackrf/ - cd hackrf - git fetch - git checkout origin/stdout - cd host - mkdir build - cd build - cmake .. -DINSTALL_UDEV_RULES=ON - make - sudo make install - """ - def has_hackrf_transfer(self): - # TODO i don't have a hackrf, so somebody doublecheck this. - # TODO also check if it has the stdout feature - return os.system("hackrf_transfer --help 2> /dev/null") != 32512 diff --git a/owrx/connection.py b/owrx/connection.py new file mode 100644 index 0000000..a0442b8 --- /dev/null +++ b/owrx/connection.py @@ -0,0 +1,180 @@ +from owrx.config import PropertyManager +from owrx.source import DspManager, CpuUsageThread, SdrService, ClientReporterThread +import json + +import logging +logger = logging.getLogger(__name__) + +class OpenWebRxClient(object): + config_keys = ["waterfall_colors", "waterfall_min_level", "waterfall_max_level", + "waterfall_auto_level_margin", "lfo_offset", "samp_rate", "fft_size", "fft_fps", + "audio_compression", "fft_compression", "max_clients", "start_mod", + "client_audio_buffer_size", "start_freq", "center_freq", "mathbox_waterfall_colors", + "mathbox_waterfall_history_length", "mathbox_waterfall_frequency_resolution"] + def __init__(self, conn): + self.conn = conn + + ClientReporterThread.getSharedInstance().addClient(self) + + self.dsp = None + self.sdr = None + self.configProps = None + + pm = PropertyManager.getSharedInstance() + + self.setSdr() + + # send receiver info + receiver_keys = ["receiver_name", "receiver_location", "receiver_qra", "receiver_asl", "receiver_gps", + "photo_title", "photo_desc"] + receiver_details = dict((key, pm.getPropertyValue(key)) for key in receiver_keys) + self.write_receiver_details(receiver_details) + + profiles = [{"name": s.getName() + " " + p["name"], "id":sid + "|" + pid} for (sid, s) in SdrService.getSources().items() for (pid, p) in s.getProfiles().items()] + self.write_profiles(profiles) + + CpuUsageThread.getSharedInstance().add_client(self) + + def sendConfig(self, key, value): + config = dict((key, self.configProps[key]) for key in OpenWebRxClient.config_keys) + # TODO mathematical properties? hmmmm + config["start_offset_freq"] = self.configProps["start_freq"] - self.configProps["center_freq"] + self.write_config(config) + def setSdr(self, id = None): + next = SdrService.getSource(id) + if (next == self.sdr): + return + + self.stopDsp() + + if self.configProps is not None: + self.configProps.unwire(self.sendConfig) + + self.sdr = next + + # send initial config + self.configProps = self.sdr.getProps().collect(*OpenWebRxClient.config_keys).defaults(PropertyManager.getSharedInstance()) + + self.configProps.wire(self.sendConfig) + self.sendConfig(None, None) + + self.sdr.addSpectrumClient(self) + + def startDsp(self): + if self.dsp is None: + self.dsp = DspManager(self, self.sdr) + self.dsp.start() + + def close(self): + self.stopDsp() + CpuUsageThread.getSharedInstance().remove_client(self) + try: + ClientReporterThread.getSharedInstance().removeClient(self) + except ValueError: + pass + logger.debug("connection closed") + + def stopDsp(self): + if self.dsp is not None: + self.dsp.stop() + self.dsp = None + if self.sdr is not None: + self.sdr.removeSpectrumClient(self) + + def setParams(self, params): + # only the keys in the protected property manager can be overridden from the web + protected = self.sdr.getProps().collect("samp_rate", "center_freq", "rf_gain", "type") \ + .defaults(PropertyManager.getSharedInstance()) + for key, value in params.items(): + protected[key] = value + + def setDspProperties(self, params): + for key, value in params.items(): + self.dsp.setProperty(key, value) + + def protected_send(self, data): + try: + self.conn.send(data) + # these exception happen when the socket is closed + except OSError: + self.close() + except ValueError: + self.close() + + def write_spectrum_data(self, data): + self.protected_send(bytes([0x01]) + data) + def write_dsp_data(self, data): + self.protected_send(bytes([0x02]) + data) + def write_s_meter_level(self, level): + self.protected_send({"type":"smeter","value":level}) + def write_cpu_usage(self, usage): + self.protected_send({"type":"cpuusage","value":usage}) + def write_clients(self, clients): + self.protected_send({"type":"clients","value":clients}) + def write_secondary_fft(self, data): + self.protected_send(bytes([0x03]) + data) + def write_secondary_demod(self, data): + self.protected_send(bytes([0x04]) + data) + def write_secondary_dsp_config(self, cfg): + self.protected_send({"type":"secondary_config", "value":cfg}) + def write_config(self, cfg): + self.protected_send({"type":"config","value":cfg}) + def write_receiver_details(self, details): + self.protected_send({"type":"receiver_details","value":details}) + def write_profiles(self, profiles): + self.protected_send({"type":"profiles","value":profiles}) + +class WebSocketMessageHandler(object): + def __init__(self): + self.handshake = None + self.client = None + self.dsp = None + + def handleTextMessage(self, conn, message): + if (message[:16] == "SERVER DE CLIENT"): + # maybe put some more info in there? nothing to store yet. + self.handshake = "completed" + logger.debug("client connection intitialized") + + self.client = OpenWebRxClient(conn) + + return + + if not self.handshake: + logger.warning("not answering client request since handshake is not complete") + return + + try: + message = json.loads(message) + if "type" in message: + if message["type"] == "dspcontrol": + if "action" in message and message["action"] == "start": + self.client.startDsp() + + if "params" in message: + params = message["params"] + self.client.setDspProperties(params) + + if message["type"] == "config": + if "params" in message: + self.client.setParams(message["params"]) + if message["type"] == "setsdr": + if "params" in message: + self.client.setSdr(message["params"]["sdr"]) + if message["type"] == "selectprofile": + if "params" in message and "profile" in message["params"]: + profile = message["params"]["profile"].split("|") + self.client.setSdr(profile[0]) + self.client.sdr.activateProfile(profile[1]) + else: + logger.warning("received message without type: {0}".format(message)) + + except json.JSONDecodeError: + logger.warning("message is not json: {0}".format(message)) + + def handleBinaryMessage(self, conn, data): + logger.error("unsupported binary message, discarding") + + def handleClose(self, conn): + if self.client: + self.client.close() diff --git a/owrx/controllers.py b/owrx/controllers.py index 366eb3a..a41015a 100644 --- a/owrx/controllers.py +++ b/owrx/controllers.py @@ -1,11 +1,11 @@ +import os import mimetypes +from datetime import datetime from owrx.websocket import WebSocketConnection from owrx.config import PropertyManager -from owrx.source import DspManager, CpuUsageThread, SdrService, ClientReporterThread +from owrx.source import ClientReporterThread +from owrx.connection import WebSocketMessageHandler from owrx.version import openwebrx_version -import json -import os -from datetime import datetime import logging logger = logging.getLogger(__name__) @@ -79,180 +79,6 @@ class IndexController(AssetsController): def handle_request(self): self.serve_file("index.wrx", "text/html") -class OpenWebRxClient(object): - config_keys = ["waterfall_colors", "waterfall_min_level", "waterfall_max_level", - "waterfall_auto_level_margin", "lfo_offset", "samp_rate", "fft_size", "fft_fps", - "audio_compression", "fft_compression", "max_clients", "start_mod", - "client_audio_buffer_size", "start_freq", "center_freq", "mathbox_waterfall_colors", - "mathbox_waterfall_history_length", "mathbox_waterfall_frequency_resolution"] - def __init__(self, conn): - self.conn = conn - - ClientReporterThread.getSharedInstance().addClient(self) - - self.dsp = None - self.sdr = None - self.configProps = None - - pm = PropertyManager.getSharedInstance() - - self.setSdr() - - # send receiver info - receiver_keys = ["receiver_name", "receiver_location", "receiver_qra", "receiver_asl", "receiver_gps", - "photo_title", "photo_desc"] - receiver_details = dict((key, pm.getPropertyValue(key)) for key in receiver_keys) - self.write_receiver_details(receiver_details) - - profiles = [{"name": s.getName() + " " + p["name"], "id":sid + "|" + pid} for (sid, s) in SdrService.getSources().items() for (pid, p) in s.getProfiles().items()] - self.write_profiles(profiles) - - CpuUsageThread.getSharedInstance().add_client(self) - - def sendConfig(self, key, value): - config = dict((key, self.configProps[key]) for key in OpenWebRxClient.config_keys) - # TODO mathematical properties? hmmmm - config["start_offset_freq"] = self.configProps["start_freq"] - self.configProps["center_freq"] - self.write_config(config) - def setSdr(self, id = None): - next = SdrService.getSource(id) - if (next == self.sdr): - return - - self.stopDsp() - - if self.configProps is not None: - self.configProps.unwire(self.sendConfig) - - self.sdr = next - - # send initial config - self.configProps = self.sdr.getProps().collect(*OpenWebRxClient.config_keys).defaults(PropertyManager.getSharedInstance()) - - self.configProps.wire(self.sendConfig) - self.sendConfig(None, None) - - self.sdr.addSpectrumClient(self) - - def startDsp(self): - if self.dsp is None: - self.dsp = DspManager(self, self.sdr) - self.dsp.start() - - def close(self): - self.stopDsp() - CpuUsageThread.getSharedInstance().remove_client(self) - try: - ClientReporterThread.getSharedInstance().removeClient(self) - except ValueError: - pass - logger.debug("connection closed") - - def stopDsp(self): - if self.dsp is not None: - self.dsp.stop() - self.dsp = None - if self.sdr is not None: - self.sdr.removeSpectrumClient(self) - - def setParams(self, params): - # only the keys in the protected property manager can be overridden from the web - protected = self.sdr.getProps().collect("samp_rate", "center_freq", "rf_gain", "type") \ - .defaults(PropertyManager.getSharedInstance()) - for key, value in params.items(): - protected[key] = value - - def setDspProperties(self, params): - for key, value in params.items(): - self.dsp.setProperty(key, value) - - def protected_send(self, data): - try: - self.conn.send(data) - # these exception happen when the socket is closed - except OSError: - self.close() - except ValueError: - self.close() - - def write_spectrum_data(self, data): - self.protected_send(bytes([0x01]) + data) - def write_dsp_data(self, data): - self.protected_send(bytes([0x02]) + data) - def write_s_meter_level(self, level): - self.protected_send({"type":"smeter","value":level}) - def write_cpu_usage(self, usage): - self.protected_send({"type":"cpuusage","value":usage}) - def write_clients(self, clients): - self.protected_send({"type":"clients","value":clients}) - def write_secondary_fft(self, data): - self.protected_send(bytes([0x03]) + data) - def write_secondary_demod(self, data): - self.protected_send(bytes([0x04]) + data) - def write_secondary_dsp_config(self, cfg): - self.protected_send({"type":"secondary_config", "value":cfg}) - def write_config(self, cfg): - self.protected_send({"type":"config","value":cfg}) - def write_receiver_details(self, details): - self.protected_send({"type":"receiver_details","value":details}) - def write_profiles(self, profiles): - self.protected_send({"type":"profiles","value":profiles}) - -class WebSocketMessageHandler(object): - def __init__(self): - self.handshake = None - self.client = None - self.dsp = None - - def handleTextMessage(self, conn, message): - if (message[:16] == "SERVER DE CLIENT"): - # maybe put some more info in there? nothing to store yet. - self.handshake = "completed" - logger.debug("client connection intitialized") - - self.client = OpenWebRxClient(conn) - - return - - if not self.handshake: - logger.warning("not answering client request since handshake is not complete") - return - - try: - message = json.loads(message) - if "type" in message: - if message["type"] == "dspcontrol": - if "action" in message and message["action"] == "start": - self.client.startDsp() - - if "params" in message: - params = message["params"] - self.client.setDspProperties(params) - - if message["type"] == "config": - if "params" in message: - self.client.setParams(message["params"]) - if message["type"] == "setsdr": - if "params" in message: - self.client.setSdr(message["params"]["sdr"]) - if message["type"] == "selectprofile": - if "params" in message and "profile" in message["params"]: - profile = message["params"]["profile"].split("|") - self.client.setSdr(profile[0]) - self.client.sdr.activateProfile(profile[1]) - else: - logger.warning("received message without type: {0}".format(message)) - - except json.JSONDecodeError: - logger.warning("message is not json: {0}".format(message)) - - def handleBinaryMessage(self, conn, data): - logger.error("unsupported binary message, discarding") - - def handleClose(self, conn): - if self.client: - self.client.close() - class WebSocketController(Controller): def handle_request(self): conn = WebSocketConnection(self.handler, WebSocketMessageHandler()) diff --git a/owrx/feature.py b/owrx/feature.py new file mode 100644 index 0000000..83f9232 --- /dev/null +++ b/owrx/feature.py @@ -0,0 +1,65 @@ +import os + +import logging +logger = logging.getLogger(__name__) + + +class UnknownFeatureException(Exception): + pass + +class FeatureDetector(object): + features = { + "core": [ "csdr", "nmux" ], + "rtl_sdr": [ "rtl_sdr" ], + "sdrplay": [ "rx_tools" ], + "hackrf": [ "hackrf_transfer" ] + } + + def is_available(self, feature): + return self.has_requirements(self.get_requirements(feature)) + + def get_requirements(self, feature): + try: + return FeatureDetector.features[feature] + except KeyError: + raise UnknownFeatureException("Feature \"{0}\" is not known.".format(feature)) + + def has_requirements(self, requirements): + passed = True + for requirement in requirements: + methodname = "has_" + requirement + if hasattr(self, methodname) and callable(getattr(self, methodname)): + passed = passed and getattr(self, methodname)() + else: + logger.error("detection of requirement {0} not implement. please fix in code!".format(requirement)) + return passed + + def has_csdr(self): + return os.system("csdr 2> /dev/null") != 32512 + + def has_nmux(self): + return os.system("nmux --help 2> /dev/null") != 32512 + + def has_rtl_sdr(self): + return os.system("rtl_sdr --help 2> /dev/null") != 32512 + + def has_rx_tools(self): + return os.system("rx_sdr --help 2> /dev/null") != 32512 + + """ + To use a HackRF, compile the HackRF host tools from its "stdout" branch: + git clone https://github.com/mossmann/hackrf/ + cd hackrf + git fetch + git checkout origin/stdout + cd host + mkdir build + cd build + cmake .. -DINSTALL_UDEV_RULES=ON + make + sudo make install + """ + def has_hackrf_transfer(self): + # TODO i don't have a hackrf, so somebody doublecheck this. + # TODO also check if it has the stdout feature + return os.system("hackrf_transfer --help 2> /dev/null") != 32512 diff --git a/owrx/source.py b/owrx/source.py index f918e54..c45d12f 100644 --- a/owrx/source.py +++ b/owrx/source.py @@ -1,5 +1,6 @@ import subprocess -from owrx.config import PropertyManager, FeatureDetector, UnknownFeatureException +from owrx.config import PropertyManager +from owrx.feature import FeatureDetector, UnknownFeatureException import threading import csdr import time From ddf9123e8b170a5ff2c7388ca2a4ba9445fcb795 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 16:02:49 +0200 Subject: [PATCH 70/73] fix auto-sqelch --- htdocs/openwebrx.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/htdocs/openwebrx.js b/htdocs/openwebrx.js index cf9f678..23f17a2 100644 --- a/htdocs/openwebrx.js +++ b/htdocs/openwebrx.js @@ -1206,7 +1206,8 @@ function on_ws_recv(evt) e('webrx-rx-photo-desc').innerHTML = r.photo_desc; break; case "smeter": - setSmeterAbsoluteValue(json.value); + smeter_level = json.value; + setSmeterAbsoluteValue(smeter_level); break; case "cpuusage": var server_cpu_usage = json.value; From 85be2e97a1b1186cabb6ca5bb97a7f097bcc33f4 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 17:20:44 +0200 Subject: [PATCH 71/73] this is now obsolete, as well --- rxws.py | 171 -------------------------------------------------------- 1 file changed, 171 deletions(-) delete mode 100644 rxws.py diff --git a/rxws.py b/rxws.py deleted file mode 100644 index a1f210c..0000000 --- a/rxws.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -rxws: WebSocket methods implemented for OpenWebRX - - This file is part of OpenWebRX, - an open-source SDR receiver software with a web UI. - Copyright (c) 2013-2015 by Andras Retzler - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -""" - -import base64 -import sha -import select -import code - -class WebSocketException(Exception): - pass - -def handshake(myself): - my_client_id=myself.path[4:] - my_headers=myself.headers.items() - my_header_keys=map(lambda x:x[0],my_headers) - h_key_exists=lambda x:my_header_keys.count(x) - h_value=lambda x:my_headers[my_header_keys.index(x)][1] - #print "The Lambdas(tm)" - #print h_key_exists("upgrade") - #print h_value("upgrade") - #print h_key_exists("sec-websocket-key") - if (not h_key_exists("upgrade")) or not (h_value("upgrade")=="websocket") or (not h_key_exists("sec-websocket-key")): - raise WebSocketException - ws_key=h_value("sec-websocket-key") - ws_key_toreturn=base64.b64encode(sha.new(ws_key+"258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()) - #A sample list of keys we get: [('origin', 'http://localhost:8073'), ('upgrade', 'websocket'), ('sec-websocket-extensions', 'x-webkit-deflate-frame'), ('sec-websocket-version', '13'), ('host', 'localhost:8073'), ('sec-websocket-key', 't9J1rgy4fc9fg2Hshhnkmg=='), ('connection', 'Upgrade'), ('pragma', 'no-cache'), ('cache-control', 'no-cache')] - myself.wfile.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "+ws_key_toreturn+"\r\nCQ-CQ-de: HA5KFU\r\n\r\n") - -def get_header(size): - #this does something similar: https://github.com/lemmingzshadow/php-websocket/blob/master/server/lib/WebSocket/Connection.php - ws_first_byte=0b10000010 # FIN=1, OP=2 - if(size>125): - ws_second_byte=126 # The following two bytes will indicate frame size - extended_size=chr((size>>8)&0xff)+chr(size&0xff) #Okay, it uses reverse byte order (little-endian) compared to anything else sent on TCP - else: - ws_second_byte=size - #256 bytes binary message in a single unmasked frame | 0x82 0x7E 0x0100 [256 bytes of binary data] - extended_size="" - return chr(ws_first_byte)+chr(ws_second_byte)+extended_size - -def code_payload(data, masking_key=""): - # both encode or decode - if masking_key=="": - key = (61, 84, 35, 6) - else: - key = [ord(i) for i in masking_key] - encoded="" - for i in range(0,len(data)): - encoded+=chr(ord(data[i])^key[i%4]) - return encoded - -def xxdg(data): - output="" - for i in range(0,len(data)/8): - output+=xxd(data[i:i+8]) - if i%2: output+="\n" - else: output+=" " - return output - - -def xxd(data): - #diagnostic purposes only - output="" - for d in data: - output+=hex(ord(d))[2:].zfill(2)+" " - return output - -#for R/W the WebSocket, use recv/send -#for reading the TCP socket, use readsock -#for writing the TCP socket, use myself.wfile.write and flush - -def readsock(myself,size,blocking): - #http://thenestofheliopolis.blogspot.hu/2011/01/how-to-implement-non-blocking-two-way.html - if blocking: - return myself.rfile.read(size) - else: - poll = select.poll() - poll.register(myself.rfile.fileno(), select.POLLIN or select.POLLPRI) - fd = poll.poll(0) #timeout is 0 - if len(fd): - f = fd[0] - if f[1] > 0: - return myself.rfile.read(size) - return "" - - -def recv(myself, blocking=False, debug=False): - bufsize=70000 - #myself.connection.setblocking(blocking) #umm... we cannot do that with rfile - if debug: print "ws_recv begin" - try: - data=readsock(myself,6,blocking) - #print "rxws.recv bytes:",xxd(data) - except: - if debug: print "ws_recv error" - return "" - if debug: print "ws_recv recved" - if(len(data)==0): return "" - fin=ord(data[0])&128!=0 - is_text_frame=ord(data[0])&15==1 - length=ord(data[1])&0x7f - data+=readsock(myself,length,blocking) - #print "rxws.recv length is ",length," (multiple packets together?) len(data) =",len(data) - has_one_byte_length=length<125 - masked=ord(data[1])&0x80!=0 - #print "len=", length, len(data)-2 - #print "fin, is_text_frame, has_one_byte_length, masked = ", (fin, is_text_frame, has_one_byte_length, masked) - #print xxd(data) - if fin and is_text_frame and has_one_byte_length: - if masked: - return code_payload(data[6:], data[2:6]) - else: - return data[2:] - -#Useful links for ideas on WebSockets: -# http://stackoverflow.com/questions/8125507/how-can-i-send-and-receive-websocket-messages-on-the-server-side -# https://developer.mozilla.org/en-US/docs/WebSockets/Writing_WebSocket_server -# http://tools.ietf.org/html/rfc6455#section-5.2 - - -def flush(myself): - myself.wfile.flush() - #or the socket, not the rfile: - #lR,lW,lX = select.select([],[myself.connection,],[],60) - - -def send(myself, data, begin_id="", debug=0): - base_frame_size=35000 #could guess by MTU? - debug=0 - #try: - while True: - counter=0 - from_end=len(data)-counter - if from_end+len(begin_id)>base_frame_size: - data_to_send=begin_id+data[counter:counter+base_frame_size-len(begin_id)] - header=get_header(len(data_to_send)) - flush(myself) - myself.wfile.write(header+data_to_send) - flush(myself) - if debug: print "rxws.send ==================== #1 if branch :: from={0} to={1} dlen={2} hlen={3}".format(counter,counter+base_frame_size-len(begin_id),len(data_to_send),len(header)) - else: - data_to_send=begin_id+data[counter:] - header=get_header(len(data_to_send)) - flush(myself) - myself.wfile.write(header+data_to_send) - flush(myself) - if debug: print "rxws.send :: #2 else branch :: dlen={0} hlen={1}".format(len(data_to_send),len(header)) - #if debug: print "header:\n"+xxdg(header)+"\n\nws data:\n"+xxdg(data_to_send) - break - counter+=base_frame_size-len(begin_id) - #except: - # pass From 17a362fe7a9393333d864c7e66e7d66baa9ae3b9 Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 17:23:03 +0200 Subject: [PATCH 72/73] no longer a template, no need for special file extension --- htdocs/{index.wrx => index.html} | 0 owrx/controllers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename htdocs/{index.wrx => index.html} (100%) diff --git a/htdocs/index.wrx b/htdocs/index.html similarity index 100% rename from htdocs/index.wrx rename to htdocs/index.html diff --git a/owrx/controllers.py b/owrx/controllers.py index a41015a..9100de4 100644 --- a/owrx/controllers.py +++ b/owrx/controllers.py @@ -77,7 +77,7 @@ class AssetsController(Controller): class IndexController(AssetsController): def handle_request(self): - self.serve_file("index.wrx", "text/html") + self.serve_file("index.html", content_type = "text/html") class WebSocketController(Controller): def handle_request(self): From a85a6c694ce008b37ab6750834b95cd78c3ccffc Mon Sep 17 00:00:00 2001 From: Jakob Ketterl Date: Sun, 12 May 2019 18:10:24 +0200 Subject: [PATCH 73/73] improve shutdown handling --- openwebrx.py | 8 ++++++-- owrx/connection.py | 10 ++++----- owrx/controllers.py | 4 ++-- owrx/sdrhu.py | 2 +- owrx/source.py | 50 +++++++++++++++++++++++++++------------------ owrx/websocket.py | 15 ++++++++++++++ 6 files changed, 58 insertions(+), 31 deletions(-) diff --git a/openwebrx.py b/openwebrx.py index e41d6c7..99b1419 100644 --- a/openwebrx.py +++ b/openwebrx.py @@ -2,7 +2,7 @@ from http.server import HTTPServer from owrx.http import RequestHandler from owrx.config import PropertyManager from owrx.feature import FeatureDetector -from owrx.source import SdrService +from owrx.source import SdrService, ClientRegistry from socketserver import ThreadingMixIn from owrx.sdrhu import SdrHuUpdater @@ -45,4 +45,8 @@ Author contact info: Andras Retzler, HA7ILM if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + for c in ClientRegistry.getSharedInstance().clients: + c.close() diff --git a/owrx/connection.py b/owrx/connection.py index a0442b8..95ce84f 100644 --- a/owrx/connection.py +++ b/owrx/connection.py @@ -1,5 +1,5 @@ from owrx.config import PropertyManager -from owrx.source import DspManager, CpuUsageThread, SdrService, ClientReporterThread +from owrx.source import DspManager, CpuUsageThread, SdrService, ClientRegistry import json import logging @@ -14,7 +14,7 @@ class OpenWebRxClient(object): def __init__(self, conn): self.conn = conn - ClientReporterThread.getSharedInstance().addClient(self) + ClientRegistry.getSharedInstance().addClient(self) self.dsp = None self.sdr = None @@ -68,10 +68,8 @@ class OpenWebRxClient(object): def close(self): self.stopDsp() CpuUsageThread.getSharedInstance().remove_client(self) - try: - ClientReporterThread.getSharedInstance().removeClient(self) - except ValueError: - pass + ClientRegistry.getSharedInstance().removeClient(self) + self.conn.close() logger.debug("connection closed") def stopDsp(self): diff --git a/owrx/controllers.py b/owrx/controllers.py index 9100de4..774ba9b 100644 --- a/owrx/controllers.py +++ b/owrx/controllers.py @@ -3,7 +3,7 @@ import mimetypes from datetime import datetime from owrx.websocket import WebSocketConnection from owrx.config import PropertyManager -from owrx.source import ClientReporterThread +from owrx.source import ClientRegistry from owrx.connection import WebSocketMessageHandler from owrx.version import openwebrx_version @@ -41,7 +41,7 @@ class StatusController(Controller): "status": "active", "name": pm["receiver_name"], "op_email": pm["receiver_admin"], - "users": ClientReporterThread.getSharedInstance().clientCount(), + "users": ClientRegistry.getSharedInstance().clientCount(), "users_max": pm["max_clients"], "gps": pm["receiver_gps"], "asl": pm["receiver_asl"], diff --git a/owrx/sdrhu.py b/owrx/sdrhu.py index b2c6b0f..5f0d7fb 100644 --- a/owrx/sdrhu.py +++ b/owrx/sdrhu.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) class SdrHuUpdater(threading.Thread): def __init__(self): self.doRun = True - super().__init__() + super().__init__(daemon = True) def update(self): pm = PropertyManager.getSharedInstance() diff --git a/owrx/source.py b/owrx/source.py index c45d12f..3efc7d4 100644 --- a/owrx/source.py +++ b/owrx/source.py @@ -467,6 +467,8 @@ class CpuUsageThread(threading.Thread): c.write_cpu_usage(cpu_usage) time.sleep(3) logger.debug("cpu usage thread shut down") + if CpuUsageThread.sharedInstance == self: + CpuUsageThread.sharedInstance = None def get_cpu_usage(self): try: @@ -499,42 +501,49 @@ class CpuUsageThread(threading.Thread): self.shutdown() def shutdown(self): - if self.doRun: - if CpuUsageThread.sharedInstance == self: - CpuUsageThread.sharedInstance = None - self.doRun = False + self.doRun = False + +class ClientReportingThread(threading.Thread): + def __init__(self, registry): + self.doRun = True + self.registry = registry + super().__init__() + def run(self): + while self.doRun: + self.registry.broadcast() + time.sleep(3) + def stop(self): + self.doRun = False class TooManyClientsException(Exception): pass -class ClientReporterThread(threading.Thread): +class ClientRegistry(object): sharedInstance = None @staticmethod def getSharedInstance(): - if ClientReporterThread.sharedInstance is None: - ClientReporterThread.sharedInstance = ClientReporterThread() - ClientReporterThread.sharedInstance.start() - ClientReporterThread.sharedInstance.doRun = True - return ClientReporterThread.sharedInstance + if ClientRegistry.sharedInstance is None: + ClientRegistry.sharedInstance = ClientRegistry() + return ClientRegistry.sharedInstance def __init__(self): - self.doRun = True self.clients = [] + self.reporter = None super().__init__() - def run(self): - while (self.doRun): - n = self.clientCount() - for c in self.clients: - c.write_clients(n) - time.sleep(3) - ClientReporterThread.sharedInstance = None + def broadcast(self): + n = self.clientCount() + for c in self.clients: + c.write_clients(n) def addClient(self, client): pm = PropertyManager.getSharedInstance() if len(self.clients) >= pm["max_clients"]: raise TooManyClientsException() self.clients.append(client) + if self.reporter is None: + self.reporter = ClientReportingThread(self) + self.reporter.start() def clientCount(self): return len(self.clients) @@ -544,5 +553,6 @@ class ClientReporterThread(threading.Thread): self.clients.remove(client) except ValueError: pass - if not self.clients: - self.doRun = False \ No newline at end of file + if not self.clients and self.reporter is not None: + self.reporter.stop() + self.reporter = None diff --git a/owrx/websocket.py b/owrx/websocket.py index 5387782..d0385b8 100644 --- a/owrx/websocket.py +++ b/owrx/websocket.py @@ -72,5 +72,20 @@ class WebSocketConnection(object): else: logger.warning("unsupported opcode: {0}".format(opcode)) + def close(self): + try: + header = self.get_header(0, 8) + self.handler.wfile.write(header) + self.handler.wfile.flush() + except ValueError: + logger.exception("while writing close frame:") + + try: + self.handler.finish() + self.handler.connection.close() + except Exception: + logger.exception("while closing connection:") + + class WebSocketException(Exception): pass