Merge branch 'wspr' into develop

This commit is contained in:
Jakob Ketterl 2019-07-14 16:48:35 +02:00
commit 0bb8b5349d
6 changed files with 160 additions and 59 deletions

19
csdr.py
View File

@ -25,7 +25,7 @@ import os
import signal import signal
import threading import threading
from functools import partial from functools import partial
from owrx.wsjt import Ft8Chopper from owrx.wsjt import Ft8Chopper, WsprChopper
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -174,7 +174,7 @@ class dsp(object):
"csdr limit_ff" "csdr limit_ff"
] ]
# fixed sample rate necessary for the wsjt-x tools. fix with sox... # fixed sample rate necessary for the wsjt-x tools. fix with sox...
if self.get_secondary_demodulator() == "ft8" and self.get_audio_rate() != self.get_output_rate(): if self.isWsjtMode() and self.get_audio_rate() != self.get_output_rate():
chain += [ chain += [
"sox -t raw -r {audio_rate} -e floating-point -b 32 -c 1 --buffer 32 - -t raw -r {output_rate} -e signed-integer -b 16 -c 1 - " "sox -t raw -r {audio_rate} -e floating-point -b 32 -c 1 --buffer 32 - -t raw -r {output_rate} -e signed-integer -b 16 -c 1 - "
] ]
@ -196,7 +196,7 @@ class dsp(object):
"csdr timing_recovery_cc GARDNER {secondary_samples_per_bits} 0.5 2 --add_q | " + \ "csdr timing_recovery_cc GARDNER {secondary_samples_per_bits} 0.5 2 --add_q | " + \
"CSDR_FIXED_BUFSIZE=1 csdr dbpsk_decoder_c_u8 | " + \ "CSDR_FIXED_BUFSIZE=1 csdr dbpsk_decoder_c_u8 | " + \
"CSDR_FIXED_BUFSIZE=1 csdr psk31_varicode_decoder_u8_u8" "CSDR_FIXED_BUFSIZE=1 csdr psk31_varicode_decoder_u8_u8"
elif which == "ft8": elif self.isWsjtMode(which):
chain = secondary_chain_base + "csdr realpart_cf | " chain = secondary_chain_base + "csdr realpart_cf | "
if self.last_decimation != 1.0 : if self.last_decimation != 1.0 :
chain += "csdr fractional_decimator_ff {last_decimation} | " chain += "csdr fractional_decimator_ff {last_decimation} | "
@ -271,8 +271,12 @@ class dsp(object):
self.secondary_processes_running = True self.secondary_processes_running = True
self.output.add_output("secondary_fft", partial(self.secondary_process_fft.stdout.read, int(self.get_secondary_fft_bytes_to_read()))) self.output.add_output("secondary_fft", partial(self.secondary_process_fft.stdout.read, int(self.get_secondary_fft_bytes_to_read())))
if self.get_secondary_demodulator() == "ft8": if self.isWsjtMode():
smd = self.get_secondary_demodulator()
if smd == "ft8":
chopper = Ft8Chopper(self.secondary_process_demod.stdout) chopper = Ft8Chopper(self.secondary_process_demod.stdout)
elif smd == "wspr":
chopper = WsprChopper(self.secondary_process_demod.stdout)
chopper.start() chopper.start()
self.output.add_output("wsjt_demod", chopper.read) self.output.add_output("wsjt_demod", chopper.read)
else: else:
@ -355,7 +359,7 @@ class dsp(object):
def get_audio_rate(self): def get_audio_rate(self):
if self.isDigitalVoice(): if self.isDigitalVoice():
return 48000 return 48000
elif self.secondary_demodulator == "ft8": elif self.isWsjtMode():
return 12000 return 12000
return self.get_output_rate() return self.get_output_rate()
@ -364,6 +368,11 @@ class dsp(object):
demodulator = self.get_demodulator() demodulator = self.get_demodulator()
return demodulator in ["dmr", "dstar", "nxdn", "ysf"] return demodulator in ["dmr", "dstar", "nxdn", "ysf"]
def isWsjtMode(self, demodulator = None):
if demodulator is None:
demodulator = self.get_secondary_demodulator()
return demodulator in ["ft8", "wspr"]
def set_output_rate(self,output_rate): def set_output_rate(self,output_rate):
self.output_rate=output_rate self.output_rate=output_rate
self.calculate_decimation() self.calculate_decimation()

View File

@ -827,7 +827,7 @@ img.openwebrx-mirror-img
} }
#openwebrx-panel-wsjt-message .message { #openwebrx-panel-wsjt-message .message {
width: 400px; width: 380px;
} }
#openwebrx-panel-wsjt-message .decimal { #openwebrx-panel-wsjt-message .decimal {
@ -835,15 +835,25 @@ img.openwebrx-mirror-img
width: 35px; width: 35px;
} }
#openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-content-container { #openwebrx-panel-wsjt-message .decimal.freq {
width: 70px;
}
#openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-content-container,
#openwebrx-panel-digimodes[data-mode="wspr"] #openwebrx-digimode-content-container
{
display: none; display: none;
} }
#openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-canvas-container { #openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-canvas-container,
#openwebrx-panel-digimodes[data-mode="wspr"] #openwebrx-digimode-canvas-container
{
height: 200px; height: 200px;
margin: -10px; margin: -10px;
} }
#openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-select-channel { #openwebrx-panel-digimodes[data-mode="ft8"] #openwebrx-digimode-select-channel,
#openwebrx-panel-digimodes[data-mode="wspr"] #openwebrx-digimode-select-channel
{
display: none; display: none;
} }

View File

@ -81,6 +81,7 @@
<option value="none"></option> <option value="none"></option>
<option value="bpsk31">BPSK31</option> <option value="bpsk31">BPSK31</option>
<option value="ft8">FT8</option> <option value="ft8">FT8</option>
<option value="wspr">WSPR</option>
</select> </select>
</div> </div>
<div class="openwebrx-panel-line"> <div class="openwebrx-panel-line">
@ -146,7 +147,7 @@
<th>UTC</th> <th>UTC</th>
<th class="decimal">dB</th> <th class="decimal">dB</th>
<th class="decimal">DT</th> <th class="decimal">DT</th>
<th class="decimal">Freq</th> <th class="decimal freq">Freq</th>
<th class="message">Message</th> <th class="message">Message</th>
</tr></thead> </tr></thead>
<tbody></tbody> <tbody></tbody>

View File

@ -1376,21 +1376,36 @@ function update_metadata(meta) {
} }
function html_escape(input) {
return $('<div/>').text(input).html()
}
function update_wsjt_panel(msg) { function update_wsjt_panel(msg) {
var $b = $('#openwebrx-panel-wsjt-message tbody'); var $b = $('#openwebrx-panel-wsjt-message tbody');
var t = new Date(msg['timestamp']); var t = new Date(msg['timestamp']);
var pad = function(i) { return ('' + i).padStart(2, "0"); } var pad = function(i) { return ('' + i).padStart(2, "0"); }
var linkedmsg = msg['msg']; var linkedmsg = msg['msg'];
if (msg['mode'] == 'FT8') {
var matches = linkedmsg.match(/(.*\s[A-Z0-9]+\s)([A-R]{2}[0-9]{2})$/); var matches = linkedmsg.match(/(.*\s[A-Z0-9]+\s)([A-R]{2}[0-9]{2})$/);
if (matches && matches[2] != 'RR73') { if (matches && matches[2] != 'RR73') {
linkedmsg = matches[1] + '<a href="/map?locator=' + matches[2] + '" target="_blank">' + matches[2] + '</a>'; linkedmsg = html_escape(matches[1]) + '<a href="/map?locator=' + matches[2] + '" target="_blank">' + matches[2] + '</a>';
} else {
linkedmsg = html_escape(linkedmsg);
}
} else if (msg['mode'] == 'WSPR') {
var matches = linkedmsg.match(/([A-Z0-9]*\s)([A-R]{2}[0-9]{2})(\s[0-9]+)/);
if (matches) {
linkedmsg = html_escape(matches[1]) + '<a href="/map?locator=' + matches[2] + '" target="_blank">' + matches[2] + '</a>' + html_escape(matches[3]);
} else {
linkedmsg = html_escape(linkedmsg);
}
} }
$b.append($( $b.append($(
'<tr data-timestamp="' + msg['timestamp'] + '">' + '<tr data-timestamp="' + msg['timestamp'] + '">' +
'<td>' + pad(t.getUTCHours()) + pad(t.getUTCMinutes()) + pad(t.getUTCSeconds()) + '</td>' + '<td>' + pad(t.getUTCHours()) + pad(t.getUTCMinutes()) + pad(t.getUTCSeconds()) + '</td>' +
'<td class="decimal">' + msg['db'] + '</td>' + '<td class="decimal">' + msg['db'] + '</td>' +
'<td class="decimal">' + msg['dt'] + '</td>' + '<td class="decimal">' + msg['dt'] + '</td>' +
'<td class="decimal">' + msg['freq'] + '</td>' + '<td class="decimal freq">' + msg['freq'] + '</td>' +
'<td class="message">' + linkedmsg + '</td>' + '<td class="message">' + linkedmsg + '</td>' +
'</tr>' '</tr>'
)); ));
@ -2693,6 +2708,7 @@ function demodulator_digital_replace(subtype)
case "bpsk31": case "bpsk31":
case "rtty": case "rtty":
case "ft8": case "ft8":
case "wspr":
secondary_demod_start(subtype); secondary_demod_start(subtype);
demodulator_analog_replace('usb', true); demodulator_analog_replace('usb', true);
demodulator_buttons_update(); demodulator_buttons_update();
@ -2700,7 +2716,7 @@ function demodulator_digital_replace(subtype)
} }
$('#openwebrx-panel-digimodes').attr('data-mode', subtype); $('#openwebrx-panel-digimodes').attr('data-mode', subtype);
toggle_panel("openwebrx-panel-digimodes", true); toggle_panel("openwebrx-panel-digimodes", true);
toggle_panel("openwebrx-panel-wsjt-message", subtype == 'ft8'); toggle_panel("openwebrx-panel-wsjt-message", ['ft8', 'wspr'].indexOf(subtype) >= 0);
} }
function secondary_demod_create_canvas() function secondary_demod_create_canvas()
@ -2863,19 +2879,16 @@ secondary_demod_listbox_updating = false;
function secondary_demod_listbox_changed() function secondary_demod_listbox_changed()
{ {
if (secondary_demod_listbox_updating) return; if (secondary_demod_listbox_updating) return;
switch ($("#openwebrx-secondary-demod-listbox")[0].value) var sdm = $("#openwebrx-secondary-demod-listbox")[0].value;
{ switch (sdm) {
case "none": case "none":
demodulator_analog_replace_last(); demodulator_analog_replace_last();
break; break;
case "bpsk31": case "bpsk31":
demodulator_digital_replace('bpsk31');
break;
case "rtty": case "rtty":
demodulator_digital_replace('rtty');
break;
case "ft8": case "ft8":
demodulator_digital_replace('ft8'); case "wspr":
demodulator_digital_replace(sdm);
break; break;
} }
} }

View File

@ -165,13 +165,15 @@ class FeatureDetector(object):
return version >= required_version return version >= required_version
except FileNotFoundError: except FileNotFoundError:
return False return False
return reduce(and_, return reduce(
and_,
map( map(
check_digiham_version, check_digiham_version,
["rrc_filter", "ysf_decoder", "dmr_decoder", "mbe_synthesizer", "gfsk_demodulator", ["rrc_filter", "ysf_decoder", "dmr_decoder", "mbe_synthesizer", "gfsk_demodulator",
"digitalvoice_filter"] "digitalvoice_filter"]
), ),
True) True
)
def has_dsd(self): def has_dsd(self):
""" """
@ -201,4 +203,11 @@ class FeatureDetector(object):
[WSJT-X homepage](https://physics.princeton.edu/pulsar/k1jt/wsjtx.html) for ready-made packages or instructions [WSJT-X homepage](https://physics.princeton.edu/pulsar/k1jt/wsjtx.html) for ready-made packages or instructions
on how to build from source. on how to build from source.
""" """
return self.command_is_runnable("jt9") return reduce(
and_,
map(
self.command_is_runnable,
["jt9", "wsprd"]
),
True
)

View File

@ -14,7 +14,7 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Ft8Chopper(threading.Thread): class WsjtChopper(threading.Thread):
def __init__(self, source): def __init__(self, source):
self.source = source self.source = source
self.tmp_dir = PropertyManager.getSharedInstance()["temporary_directory"] self.tmp_dir = PropertyManager.getSharedInstance()["temporary_directory"]
@ -27,7 +27,7 @@ class Ft8Chopper(threading.Thread):
super().__init__() super().__init__()
def getWaveFile(self): def getWaveFile(self):
filename = "{tmp_dir}/openwebrx-ft8chopper-{id}-{timestamp}.wav".format( filename = "{tmp_dir}/openwebrx-wsjtchopper-{id}-{timestamp}.wav".format(
tmp_dir = self.tmp_dir, tmp_dir = self.tmp_dir,
id = id(self), id = id(self),
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
@ -40,11 +40,10 @@ class Ft8Chopper(threading.Thread):
def getNextDecodingTime(self): def getNextDecodingTime(self):
t = datetime.now() t = datetime.now()
seconds = (int(t.second / 15) + 1) * 15 zeroed = t.replace(minute=0, second=0, microsecond=0)
if seconds >= 60: delta = t - zeroed
t = t + timedelta(minutes = 1) seconds = (int(delta.total_seconds() / self.interval) + 1) * self.interval
seconds = 0 t = zeroed + timedelta(seconds = seconds)
t = t.replace(second = seconds, microsecond = 0)
logger.debug("scheduling: {0}".format(t)) logger.debug("scheduling: {0}".format(t))
return t.timestamp() return t.timestamp()
@ -70,10 +69,15 @@ class Ft8Chopper(threading.Thread):
self.fileQueue.append(filename) self.fileQueue.append(filename)
self._scheduleNextSwitch() self._scheduleNextSwitch()
def decoder_commandline(self, file):
'''
must be overridden in child classes
'''
return []
def decode(self): def decode(self):
def decode_and_unlink(file): def decode_and_unlink(file):
#TODO expose decoding quality parameters through config decoder = subprocess.Popen(self.decoder_commandline(file), stdout=subprocess.PIPE, cwd=self.tmp_dir)
decoder = subprocess.Popen(["jt9", "--ft8", "-d", "3", file], stdout=subprocess.PIPE, cwd=self.tmp_dir)
while True: while True:
line = decoder.stdout.readline() line = decoder.stdout.readline()
if line is None or (isinstance(line, bytes) and len(line) == 0): if line is None or (isinstance(line, bytes) and len(line) == 0):
@ -91,12 +95,12 @@ class Ft8Chopper(threading.Thread):
threading.Thread(target=decode_and_unlink, args=[file]).start() threading.Thread(target=decode_and_unlink, args=[file]).start()
def run(self) -> None: def run(self) -> None:
logger.debug("FT8 chopper starting up") logger.debug("WSJT chopper starting up")
self.startScheduler() self.startScheduler()
while self.doRun: while self.doRun:
data = self.source.read(256) data = self.source.read(256)
if data is None or (isinstance(data, bytes) and len(data) == 0): if data is None or (isinstance(data, bytes) and len(data) == 0):
logger.warning("zero read on ft8 chopper") logger.warning("zero read on WSJT chopper")
self.doRun = False self.doRun = False
else: else:
self.switchingLock.acquire() self.switchingLock.acquire()
@ -104,7 +108,7 @@ class Ft8Chopper(threading.Thread):
self.switchingLock.release() self.switchingLock.release()
self.decode() self.decode()
logger.debug("FT8 chopper shutting down") logger.debug("WSJT chopper shutting down")
self.outputReader.close() self.outputReader.close()
self.outputWriter.close() self.outputWriter.close()
self.emptyScheduler() self.emptyScheduler()
@ -120,10 +124,34 @@ class Ft8Chopper(threading.Thread):
return None return None
class Ft8Chopper(WsjtChopper):
def __init__(self, source):
self.interval = 15
super().__init__(source)
def decoder_commandline(self, file):
#TODO expose decoding quality parameters through config
return ["jt9", "--ft8", "-d", "3", file]
class WsprChopper(WsjtChopper):
def __init__(self, source):
self.interval = 120
super().__init__(source)
def decoder_commandline(self, file):
#TODO expose decoding quality parameters through config
return ["wsprd", "-d", file]
class WsjtParser(object): class WsjtParser(object):
locator_pattern = re.compile(".*\\s([A-Z0-9]+)\\s([A-R]{2}[0-9]{2})$")
jt9_pattern = re.compile("^[0-9]{6} .*")
wspr_pattern = re.compile("^[0-9]{4} .*")
wspr_splitter_pattern = re.compile("([A-Z0-9]*)\\s([A-R]{2}[0-9]{2})\\s([0-9]+)")
def __init__(self, handler): def __init__(self, handler):
self.handler = handler self.handler = handler
self.locator_pattern = re.compile(".*\s([A-Z0-9]+)\s([A-R]{2}[0-9]{2})$")
modes = { modes = {
"~": "FT8" "~": "FT8"
@ -132,14 +160,25 @@ class WsjtParser(object):
def parse(self, data): def parse(self, data):
try: try:
msg = data.decode().rstrip() msg = data.decode().rstrip()
# sample
# '222100 -15 -0.0 508 ~ CQ EA7MJ IM66'
# known debug messages we know to skip # known debug messages we know to skip
if msg.startswith("<DecodeFinished>"): if msg.startswith("<DecodeFinished>"):
return return
if msg.startswith(" EOF on input file"): if msg.startswith(" EOF on input file"):
return return
out = {}
if WsjtParser.jt9_pattern.match(msg):
out = self.parse_from_jt9(msg)
elif WsjtParser.wspr_pattern.match(msg):
out = self.parse_from_wsprd(msg)
self.handler.write_wsjt_message(out)
except ValueError:
logger.exception("error while parsing wsjt message")
def parse_from_jt9(self, msg):
# ft8 sample
# '222100 -15 -0.0 508 ~ CQ EA7MJ IM66'
out = {} out = {}
ts = datetime.strptime(msg[0:6], "%H%M%S") ts = datetime.strptime(msg[0:6], "%H%M%S")
out["timestamp"] = int(datetime.combine(date.today(), ts.time(), datetime.now().tzinfo).timestamp() * 1000) out["timestamp"] = int(datetime.combine(date.today(), ts.time(), datetime.now().tzinfo).timestamp() * 1000)
@ -151,13 +190,10 @@ class WsjtParser(object):
wsjt_msg = msg[24:60].strip() wsjt_msg = msg[24:60].strip()
self.parseLocator(wsjt_msg, mode) self.parseLocator(wsjt_msg, mode)
out["msg"] = wsjt_msg out["msg"] = wsjt_msg
return out
self.handler.write_wsjt_message(out)
except ValueError:
logger.exception("error while parsing wsjt message")
def parseLocator(self, msg, mode): def parseLocator(self, msg, mode):
m = self.locator_pattern.match(msg) m = WsjtParser.locator_pattern.match(msg)
if m is None: if m is None:
return return
# this is a valid locator in theory, but it's somewhere in the arctic ocean, near the north pole, so it's very # this is a valid locator in theory, but it's somewhere in the arctic ocean, near the north pole, so it's very
@ -165,3 +201,26 @@ class WsjtParser(object):
if m.group(2) == "RR73": if m.group(2) == "RR73":
return return
Map.getSharedInstance().updateLocation(m.group(1), LocatorLocation(m.group(2)), mode) Map.getSharedInstance().updateLocation(m.group(1), LocatorLocation(m.group(2)), mode)
def parse_from_wsprd(self, msg):
# wspr sample
# '2600 -24 0.4 0.001492 -1 G8AXA JO01 33'
out = {}
now = datetime.now()
ts = datetime.strptime(msg[0:4], "%M%S").replace(hour=now.hour)
out["timestamp"] = int(datetime.combine(date.today(), ts.time(), now.tzinfo).timestamp() * 1000)
out["db"] = float(msg[5:8])
out["dt"] = float(msg[9:13])
out["freq"] = float(msg[14:24])
out["drift"] = int(msg[25:28])
out["mode"] = "WSPR"
wsjt_msg = msg[29:].strip()
out["msg"] = wsjt_msg
self.parseWsprMessage(wsjt_msg)
return out
def parseWsprMessage(self, msg):
m = WsjtParser.wspr_splitter_pattern.match(msg)
if m is None:
return
Map.getSharedInstance().updateLocation(m.group(1), LocatorLocation(m.group(2)), "WSPR")