591 lines
22 KiB
Python
591 lines
22 KiB
Python
from owrx.map import Map, LatLngLocation
|
|
from owrx.metrics import Metrics, CounterMetric
|
|
from owrx.bands import Bandplan
|
|
from datetime import datetime, timezone
|
|
from csdr.module import PickleModule
|
|
import re
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# speed is in knots... convert to metric (km/h)
|
|
knotsToKilometers = 1.852
|
|
feetToMeters = 0.3048
|
|
milesToKilometers = 1.609344
|
|
inchesToMilimeters = 25.4
|
|
|
|
|
|
def fahrenheitToCelsius(f):
|
|
return (f - 32) * 5 / 9
|
|
|
|
|
|
# not sure what the correct encoding is. it seems TAPR has set utf-8 as a standard, but not everybody is following it.
|
|
encoding = "utf-8"
|
|
|
|
# regex for altitute in comment field
|
|
altitudeRegex = re.compile("(^.*)\\/A=([0-9]{6})(.*$)")
|
|
|
|
# regex for parsing third-party headers
|
|
thirdpartyeRegex = re.compile("^([a-zA-Z0-9-]+)>((([a-zA-Z0-9-]+\\*?,)*)([a-zA-Z0-9-]+\\*?)):(.*)$")
|
|
|
|
# regex for getting the message id out of message
|
|
messageIdRegex = re.compile("^(.*){([0-9]{1,5})$")
|
|
|
|
# regex to filter pseudo "WIDE" path elements
|
|
widePattern = re.compile("^WIDE[0-9]-[0-9]$")
|
|
|
|
|
|
def decodeBase91(input):
|
|
base = decodeBase91(input[:-1]) * 91 if len(input) > 1 else 0
|
|
return base + (ord(input[-1]) - 33)
|
|
|
|
|
|
def getSymbolData(symbol, table):
|
|
return {"symbol": symbol, "table": table, "index": ord(symbol) - 33, "tableindex": ord(table) - 33}
|
|
|
|
|
|
class Ax25Parser(PickleModule):
|
|
def process(self, ax25frame):
|
|
control_pid = ax25frame.find(bytes([0x03, 0xF0]))
|
|
if control_pid % 7 > 0:
|
|
logger.warning("aprs packet framing error: control/pid position not aligned with 7-octet callsign data")
|
|
|
|
def chunks(l, n):
|
|
"""Yield successive n-sized chunks from l."""
|
|
for i in range(0, len(l), n):
|
|
yield l[i:i + n]
|
|
|
|
try:
|
|
return {
|
|
"destination": self.extractCallsign(ax25frame[0:7]),
|
|
"source": self.extractCallsign(ax25frame[7:14]),
|
|
"path": [self.extractCallsign(c) for c in chunks(ax25frame[14:control_pid], 7)],
|
|
"data": ax25frame[control_pid + 2 :],
|
|
}
|
|
except (ValueError, IndexError):
|
|
logger.exception("error parsing ax25 frame")
|
|
|
|
def extractCallsign(self, input):
|
|
cs = bytes([b >> 1 for b in input[0:6]]).decode(encoding, "replace").strip()
|
|
ssid = (input[6] & 0b00011110) >> 1
|
|
if ssid > 0:
|
|
return "{callsign}-{ssid}".format(callsign=cs, ssid=ssid)
|
|
else:
|
|
return cs
|
|
|
|
|
|
class WeatherMapping(object):
|
|
def __init__(self, char, key, length, scale=None):
|
|
self.char = char
|
|
self.key = key
|
|
self.length = length
|
|
self.scale = scale
|
|
|
|
def matches(self, input):
|
|
return self.char == input[0] and len(input) > self.length
|
|
|
|
def updateWeather(self, weather, input):
|
|
def deepApply(obj, key, v):
|
|
keys = key.split(".")
|
|
if len(keys) > 1:
|
|
if not keys[0] in obj:
|
|
obj[keys[0]] = {}
|
|
deepApply(obj[keys[0]], ".".join(keys[1:]), v)
|
|
else:
|
|
obj[key] = v
|
|
|
|
try:
|
|
value = int(input[1 : 1 + self.length])
|
|
if self.scale:
|
|
value = self.scale(value)
|
|
deepApply(weather, self.key, value)
|
|
except ValueError:
|
|
pass
|
|
remain = input[1 + self.length :]
|
|
return weather, remain
|
|
|
|
|
|
class WeatherParser(object):
|
|
mappings = [
|
|
WeatherMapping("c", "wind.direction", 3),
|
|
WeatherMapping("s", "wind.speed", 3, lambda x: x * milesToKilometers),
|
|
WeatherMapping("g", "wind.gust", 3, lambda x: x * milesToKilometers),
|
|
WeatherMapping("t", "temperature", 3, fahrenheitToCelsius),
|
|
WeatherMapping("r", "rain.hour", 3, lambda x: x / 100 * inchesToMilimeters),
|
|
WeatherMapping("p", "rain.day", 3, lambda x: x / 100 * inchesToMilimeters),
|
|
WeatherMapping("P", "rain.sincemidnight", 3, lambda x: x / 100 * inchesToMilimeters),
|
|
WeatherMapping("h", "humidity", 2),
|
|
WeatherMapping("b", "barometricpressure", 5, lambda x: x / 10),
|
|
WeatherMapping("s", "snowfall", 3, lambda x: x * 25.4),
|
|
]
|
|
|
|
def __init__(self, data, weather=None):
|
|
self.data = data
|
|
self.weather = {} if weather is None else weather
|
|
|
|
def getWeather(self):
|
|
doWork = True
|
|
weather = self.weather
|
|
while doWork:
|
|
mapping = next((m for m in WeatherParser.mappings if m.matches(self.data)), None)
|
|
if mapping:
|
|
(weather, remain) = mapping.updateWeather(weather, self.data)
|
|
self.data = remain
|
|
doWork = len(self.data) > 0
|
|
else:
|
|
doWork = False
|
|
return weather
|
|
|
|
def getRemainder(self):
|
|
return self.data
|
|
|
|
|
|
class AprsLocation(LatLngLocation):
|
|
def __init__(self, data):
|
|
super().__init__(data["lat"], data["lon"])
|
|
self.data = data
|
|
|
|
def __dict__(self):
|
|
res = super(AprsLocation, self).__dict__()
|
|
for key in ["comment", "symbol", "course", "speed", "altitude", "weather", "device", "power", "height", "gain", "directivity"]:
|
|
if key in self.data:
|
|
res[key] = self.data[key]
|
|
return res
|
|
|
|
|
|
class AprsParser(PickleModule):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.metrics = {}
|
|
self.band = None
|
|
|
|
def setDialFrequency(self, freq):
|
|
self.band = Bandplan.getSharedInstance().findBand(freq)
|
|
|
|
def getMetric(self, category):
|
|
if category not in self.metrics:
|
|
band = "unknown"
|
|
if self.band is not None:
|
|
band = self.band.getName()
|
|
name = "aprs.decodes.{band}.aprs.{category}".format(band=band, category=category)
|
|
metrics = Metrics.getSharedInstance()
|
|
self.metrics[category] = metrics.getMetric(name)
|
|
if self.metrics[category] is None:
|
|
self.metrics[category] = CounterMetric()
|
|
metrics.addMetric(name, self.metrics[category])
|
|
return self.metrics[category]
|
|
|
|
def isDirect(self, aprsData):
|
|
if "path" in aprsData and len(aprsData["path"]) > 0:
|
|
hops = [host for host in aprsData["path"] if widePattern.match(host) is None]
|
|
if len(hops) > 0:
|
|
return False
|
|
if "type" in aprsData and aprsData["type"] in ["thirdparty", "item", "object"]:
|
|
return False
|
|
return True
|
|
|
|
def process(self, data):
|
|
try:
|
|
# TODO how can we tell if this is an APRS frame at all?
|
|
aprsData = self.parseAprsData(data)
|
|
|
|
logger.debug("decoded APRS data: %s", aprsData)
|
|
self.updateMap(aprsData)
|
|
self.getMetric("total").inc()
|
|
if self.isDirect(aprsData):
|
|
self.getMetric("direct").inc()
|
|
|
|
# the frontend uses this to distinguis hessages from the different parsers
|
|
aprsData["mode"] = "APRS"
|
|
return aprsData
|
|
except Exception:
|
|
logger.exception("exception while parsing aprs data")
|
|
|
|
def updateMap(self, mapData):
|
|
if "type" in mapData and mapData["type"] == "thirdparty" and "data" in mapData:
|
|
mapData = mapData["data"]
|
|
if "lat" in mapData and "lon" in mapData:
|
|
loc = AprsLocation(mapData)
|
|
source = mapData["source"]
|
|
if "type" in mapData:
|
|
if mapData["type"] == "item":
|
|
source = mapData["item"]
|
|
elif mapData["type"] == "object":
|
|
source = mapData["object"]
|
|
Map.getSharedInstance().updateLocation(source, loc, "APRS", self.band)
|
|
|
|
def hasCompressedCoordinates(self, raw):
|
|
return raw[0] == "/" or raw[0] == "\\"
|
|
|
|
def parseUncompressedCoordinates(self, raw):
|
|
lat = int(raw[0:2]) + float(raw[2:7]) / 60
|
|
if raw[7] == "S":
|
|
lat *= -1
|
|
lon = int(raw[9:12]) + float(raw[12:17]) / 60
|
|
if raw[17] == "W":
|
|
lon *= -1
|
|
return {"lat": lat, "lon": lon, "symbol": getSymbolData(raw[18], raw[8])}
|
|
|
|
def parseCompressedCoordinates(self, raw):
|
|
return {
|
|
"lat": 90 - decodeBase91(raw[1:5]) / 380926,
|
|
"lon": -180 + decodeBase91(raw[5:9]) / 190463,
|
|
"symbol": getSymbolData(raw[9], raw[0]),
|
|
}
|
|
|
|
def parseTimestamp(self, raw):
|
|
now = datetime.now()
|
|
if raw[6] == "h":
|
|
ts = datetime.strptime(raw[0:6], "%H%M%S")
|
|
ts = ts.replace(year=now.year, month=now.month, day=now.month, tzinfo=timezone.utc)
|
|
else:
|
|
ts = datetime.strptime(raw[0:6], "%d%H%M")
|
|
ts = ts.replace(year=now.year, month=now.month)
|
|
if raw[6] == "z":
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
elif raw[6] == "/":
|
|
ts = ts.replace(tzinfo=now.tzinfo)
|
|
else:
|
|
logger.warning("invalid timezone info byte: %s", raw[6])
|
|
return int(ts.timestamp() * 1000)
|
|
|
|
def parseStatusUpate(self, raw):
|
|
res = {"type": "status"}
|
|
if raw[6] == "z":
|
|
res["timestamp"] = self.parseTimestamp(raw[0:7])
|
|
res["comment"] = raw[7:]
|
|
else:
|
|
res["comment"] = raw
|
|
return res
|
|
|
|
def parseAprsData(self, data):
|
|
information = data["data"]
|
|
|
|
# forward some of the ax25 data
|
|
aprsData = {"source": data["source"], "destination": data["destination"], "path": data["path"]}
|
|
|
|
if information[0] == 0x1C or information[0] == ord("`") or information[0] == ord("'"):
|
|
aprsData.update(MicEParser().parse(data))
|
|
return aprsData
|
|
|
|
information = information.decode(encoding, "replace")
|
|
|
|
# APRS data type identifier
|
|
dti = information[0]
|
|
|
|
if dti == "!" or dti == "=":
|
|
# position without timestamp
|
|
aprsData.update(self.parseRegularAprsData(information[1:]))
|
|
elif dti == "/" or dti == "@":
|
|
# position with timestamp
|
|
aprsData["timestamp"] = self.parseTimestamp(information[1:8])
|
|
aprsData.update(self.parseRegularAprsData(information[8:]))
|
|
elif dti == ">":
|
|
# status update
|
|
aprsData.update(self.parseStatusUpate(information[1:]))
|
|
elif dti == "}":
|
|
# third party
|
|
aprsData.update(self.parseThirdpartyAprsData(information[1:]))
|
|
elif dti == ":":
|
|
# message
|
|
aprsData.update(self.parseMessage(information[1:]))
|
|
elif dti == ";":
|
|
# object
|
|
aprsData.update(self.parseObject(information[1:]))
|
|
elif dti == ")":
|
|
# item
|
|
aprsData.update(self.parseItem(information[1:]))
|
|
|
|
return aprsData
|
|
|
|
def parseObject(self, information):
|
|
result = {"type": "object"}
|
|
if len(information) > 16:
|
|
result["object"] = information[0:9].strip()
|
|
result["live"] = information[9] == "*"
|
|
result["timestamp"] = self.parseTimestamp(information[10:17])
|
|
result.update(self.parseRegularAprsData(information[17:]))
|
|
# override type, losing information about compression
|
|
result["type"] = "object"
|
|
return result
|
|
|
|
def parseItem(self, information):
|
|
result = {"type": "item"}
|
|
if len(information) > 3:
|
|
indexes = [information[0:10].find(p) for p in ["!", "_"]]
|
|
filtered = [i for i in indexes if i >= 3]
|
|
filtered.sort()
|
|
if len(filtered):
|
|
index = filtered[0]
|
|
result["item"] = information[0:index]
|
|
result["live"] = information[index] == "!"
|
|
result.update(self.parseRegularAprsData(information[index + 1 :]))
|
|
# override type, losing information about compression
|
|
result["type"] = "item"
|
|
return result
|
|
|
|
def parseMessage(self, information):
|
|
result = {"type": "message"}
|
|
if len(information) > 9 and information[9] == ":":
|
|
result["adressee"] = information[0:9]
|
|
message = information[10:]
|
|
if len(message) > 3 and message[0:3] == "ack":
|
|
result["type"] = "messageacknowledgement"
|
|
result["messageid"] = int(message[3:8])
|
|
elif len(message) > 3 and message[0:3] == "rej":
|
|
result["type"] = "messagerejection"
|
|
result["messageid"] = int(message[3:8])
|
|
else:
|
|
matches = messageIdRegex.match(message)
|
|
if matches:
|
|
result["messageid"] = int(matches.group(2))
|
|
message = matches.group(1)
|
|
result["message"] = message
|
|
return result
|
|
|
|
def parseThirdpartyAprsData(self, information):
|
|
matches = thirdpartyeRegex.match(information)
|
|
if matches:
|
|
path = matches.group(2).split(",")
|
|
destination = next((c.strip("*").upper() for c in path if c.endswith("*")), None)
|
|
data = self.parseAprsData(
|
|
{
|
|
"source": matches.group(1).upper(),
|
|
"destination": destination,
|
|
"path": path,
|
|
"data": matches.group(6).encode(encoding),
|
|
}
|
|
)
|
|
return {"type": "thirdparty", "data": data}
|
|
|
|
return {"type": "thirdparty"}
|
|
|
|
def parseRegularAprsData(self, information):
|
|
if self.hasCompressedCoordinates(information):
|
|
aprsData = self.parseCompressedCoordinates(information[0:10])
|
|
aprsData["type"] = "compressed"
|
|
if information[10] != " ":
|
|
if information[10] == "{":
|
|
# pre-calculated radio range
|
|
aprsData["range"] = 2 * 1.08 ** (ord(information[11]) - 33) * milesToKilometers
|
|
else:
|
|
aprsData["course"] = (ord(information[10]) - 33) * 4
|
|
# speed is in knots... convert to metric (km/h)
|
|
aprsData["speed"] = (1.08 ** (ord(information[11]) - 33) - 1) * knotsToKilometers
|
|
# compression type
|
|
t = ord(information[12])
|
|
aprsData["fix"] = (t & 0b00100000) > 0
|
|
sources = ["other", "GLL", "GGA", "RMC"]
|
|
aprsData["nmeasource"] = sources[(t & 0b00011000) >> 3]
|
|
origins = [
|
|
"Compressed",
|
|
"TNC BText",
|
|
"Software",
|
|
"[tbd]",
|
|
"KPC3",
|
|
"Pico",
|
|
"Other tracker",
|
|
"Digipeater conversion",
|
|
]
|
|
aprsData["compressionorigin"] = origins[t & 0b00000111]
|
|
comment = information[13:]
|
|
else:
|
|
aprsData = self.parseUncompressedCoordinates(information[0:19])
|
|
aprsData["type"] = "regular"
|
|
comment = information[19:]
|
|
|
|
def decodeHeightGainDirectivity(comment):
|
|
res = {"height": 2 ** int(comment[4]) * 10 * feetToMeters, "gain": int(comment[5])}
|
|
directivity = int(comment[6])
|
|
if directivity == 0:
|
|
res["directivity"] = "omni"
|
|
elif 0 < directivity < 9:
|
|
res["directivity"] = directivity * 45
|
|
return res
|
|
|
|
# aprs data extensions
|
|
# yes, weather stations are officially identified by their symbols. go figure...
|
|
if "symbol" in aprsData and aprsData["symbol"]["index"] == 62:
|
|
# weather report
|
|
weather = {}
|
|
if len(comment) > 6 and comment[3] == "/":
|
|
try:
|
|
weather["wind"] = {"direction": int(comment[0:3]), "speed": int(comment[4:7]) * milesToKilometers}
|
|
except ValueError:
|
|
pass
|
|
comment = comment[7:]
|
|
|
|
parser = WeatherParser(comment, weather)
|
|
aprsData["weather"] = parser.getWeather()
|
|
comment = parser.getRemainder()
|
|
elif len(comment) > 6:
|
|
if comment[3] == "/":
|
|
# course and speed
|
|
# for a weather report, this would be wind direction and speed
|
|
try:
|
|
aprsData["course"] = int(comment[0:3])
|
|
aprsData["speed"] = int(comment[4:7]) * knotsToKilometers
|
|
except ValueError:
|
|
pass
|
|
comment = comment[7:]
|
|
elif comment[0:3] == "PHG":
|
|
# station power and effective antenna height/gain/directivity
|
|
try:
|
|
powerCodes = [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
|
|
aprsData["power"] = powerCodes[int(comment[3])]
|
|
aprsData.update(decodeHeightGainDirectivity(comment))
|
|
except ValueError:
|
|
pass
|
|
comment = comment[7:]
|
|
elif comment[0:3] == "RNG":
|
|
# pre-calculated radio range
|
|
try:
|
|
aprsData["range"] = int(comment[3:7]) * milesToKilometers
|
|
except ValueError:
|
|
pass
|
|
comment = comment[7:]
|
|
elif comment[0:3] == "DFS":
|
|
# direction finding signal strength and antenna height/gain
|
|
try:
|
|
aprsData["strength"] = int(comment[3])
|
|
aprsData.update(decodeHeightGainDirectivity(comment))
|
|
except ValueError:
|
|
pass
|
|
comment = comment[7:]
|
|
|
|
matches = altitudeRegex.match(comment)
|
|
if matches:
|
|
aprsData["altitude"] = int(matches.group(2)) * feetToMeters
|
|
comment = matches.group(1) + matches.group(3)
|
|
|
|
aprsData["comment"] = comment
|
|
|
|
return aprsData
|
|
|
|
|
|
class MicEParser(object):
|
|
def extractNumber(self, input):
|
|
n = ord(input)
|
|
if n >= ord("P"):
|
|
return n - ord("P")
|
|
if n >= ord("A"):
|
|
return n - ord("A")
|
|
return n - ord("0")
|
|
|
|
def listToNumber(self, input):
|
|
base = self.listToNumber(input[:-1]) * 10 if len(input) > 1 else 0
|
|
return base + input[-1]
|
|
|
|
def extractAltitude(self, comment):
|
|
if len(comment) < 4 or comment[3] != "}":
|
|
return (comment, None)
|
|
return comment[4:], decodeBase91(comment[:3]) - 10000
|
|
|
|
def extractDevice(self, comment):
|
|
if len(comment) > 0:
|
|
if comment[0] == ">":
|
|
if len(comment) > 1:
|
|
if comment[-1] == "=":
|
|
return comment[1:-1], {"manufacturer": "Kenwood", "device": "TH-D72"}
|
|
if comment[-1] == "^":
|
|
return comment[1:-1], {"manufacturer": "Kenwood", "device": "TH-D74"}
|
|
return comment[1:], {"manufacturer": "Kenwood", "device": "TH-D7A"}
|
|
if comment[0] == "]":
|
|
if len(comment) > 1 and comment[-1] == "=":
|
|
return comment[1:-1], {"manufacturer": "Kenwood", "device": "TM-D710"}
|
|
return comment[1:], {"manufacturer": "Kenwood", "device": "TM-D700"}
|
|
if len(comment) > 2 and (comment[0] == "`" or comment[0] == "'"):
|
|
if comment[-2] == "_":
|
|
devices = {
|
|
"b": "VX-8",
|
|
'"': "FTM-350",
|
|
"#": "VX-8G",
|
|
"$": "FT1D",
|
|
"%": "FTM-400DR",
|
|
")": "FTM-100D",
|
|
"(": "FT2D",
|
|
"0": "FT3D",
|
|
}
|
|
return comment[1:-2], {"manufacturer": "Yaesu", "device": devices.get(comment[-1], "Unknown")}
|
|
if comment[-2:] == " X":
|
|
return comment[1:-2], {"manufacturer": "SainSonic", "device": "AP510"}
|
|
if comment[-2] == "(":
|
|
devices = {"5": "D578UV", "8": "D878UV"}
|
|
return comment[1:-2], {"manufacturer": "Anytone", "device": devices.get(comment[-1], "Unknown")}
|
|
if comment[-2] == "|":
|
|
devices = {"3": "TinyTrack3", "4": "TinyTrack4"}
|
|
return comment[1:-2], {"manufacturer": "Byonics", "device": devices.get(comment[-1], "Unknown")}
|
|
if comment[-2:] == "^v":
|
|
return comment[1:-2], {"manufacturer": "HinzTec", "device": "anyfrog"}
|
|
if comment[-2] == ":":
|
|
devices = {"4": "P4dragon DR-7400 modem", "8": "P4dragon DR-7800 modem"}
|
|
return (
|
|
comment[1:-2],
|
|
{"manufacturer": "SCS GmbH & Co.", "device": devices.get(comment[-1], "Unknown")},
|
|
)
|
|
if comment[-2:] == "~v":
|
|
return comment[1:-2], {"manufacturer": "Other", "device": "Other"}
|
|
return comment[1:-2], None
|
|
return comment, None
|
|
|
|
def parse(self, data):
|
|
information = data["data"]
|
|
destination = data["destination"]
|
|
|
|
rawLatitude = [self.extractNumber(c) for c in destination[0:6]]
|
|
lat = self.listToNumber(rawLatitude[0:2]) + self.listToNumber(rawLatitude[2:6]) / 6000
|
|
if ord(destination[3]) <= ord("9"):
|
|
lat *= -1
|
|
|
|
lon = information[1] - 28
|
|
if ord(destination[4]) >= ord("P"):
|
|
lon += 100
|
|
if 180 <= lon <= 189:
|
|
lon -= 80
|
|
if 190 <= lon <= 199:
|
|
lon -= 190
|
|
|
|
minutes = information[2] - 28
|
|
if minutes >= 60:
|
|
minutes -= 60
|
|
|
|
lon += minutes / 60 + (information[3] - 28) / 6000
|
|
|
|
if ord(destination[5]) >= ord("P"):
|
|
lon *= -1
|
|
|
|
speed = (information[4] - 28) * 10
|
|
dc28 = information[5] - 28
|
|
speed += int(dc28 / 10)
|
|
course = (dc28 % 10) * 100
|
|
course += information[6] - 28
|
|
if speed >= 800:
|
|
speed -= 800
|
|
if course >= 400:
|
|
course -= 400
|
|
# speed is in knots... convert to metric (km/h)
|
|
speed *= knotsToKilometers
|
|
|
|
comment = information[9:].decode(encoding, "replace").strip()
|
|
(comment, altitude) = self.extractAltitude(comment)
|
|
|
|
(comment, device) = self.extractDevice(comment)
|
|
|
|
# altitude might be inside the device string, so repeat and choose one
|
|
(comment, insideAltitude) = self.extractAltitude(comment)
|
|
altitude = next((a for a in [altitude, insideAltitude] if a is not None), None)
|
|
|
|
return {
|
|
"fix": information[0] == ord("`") or information[0] == 0x1C,
|
|
"lat": lat,
|
|
"lon": lon,
|
|
"comment": comment,
|
|
"altitude": altitude,
|
|
"speed": speed,
|
|
"course": course,
|
|
"device": device,
|
|
"type": "Mic-E",
|
|
"symbol": getSymbolData(chr(information[7]), chr(information[8])),
|
|
}
|