joachim/openwebrx-clone
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement config layering

Browse Source
This commit is contained in:
Jakob Ketterl 2021-02-11 19:31:44 +01:00
parent e926611307
commit f23fa59ac3
20 changed files with 524 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
owrx/__main__.py
View File

@ -1,6 +1,7 @@
from http.server import HTTPServer
from owrx.http import RequestHandler
from owrx.config import Config, CoreConfig
from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.feature import FeatureDetector
from owrx.sdr import SdrService
from socketserver import ThreadingMixIn

3
owrx/audio.py
View File

@ -1,5 +1,6 @@
from abc import ABC, ABCMeta, abstractmethod
from owrx.config import Config, CoreConfig
from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.metrics import Metrics, CounterMetric, DirectMetric
import threading
import wave

149
owrx/config/__init__.py
View File

@ -1,129 +1,32 @@
from configparser import ConfigParser
from owrx.property import PropertyLayer
import importlib.util
import os
import json
from glob import glob
from owrx.config.error import ConfigError, ConfigNotFoundException
from owrx.config.migration import ConfigMigratorVersion1, ConfigMigratorVersion2
import logging
logger = logging.getLogger(__name__)
from owrx.property import PropertyStack
from owrx.config.error import ConfigError
from owrx.config.defaults import defaultConfig
from owrx.config.dynamic import DynamicConfig
from owrx.config.classic import ClassicConfig
class CoreConfig(object):
defaults = {
"core": {
"data_directory": "/var/lib/openwebrx",
"temporary_directory": "/tmp",
},
"web": {
"port": 8073,
},
"aprs": {
"symbols_path": "/usr/share/aprs-symbols/png"
}
}
class Config(PropertyStack):
sharedConfig = None
def __init__(self):
config = ConfigParser()
# set up config defaults
config.read_dict(CoreConfig.defaults)
# check for overrides
overrides_dir = "/etc/openwebrx/openwebrx.conf.d"
if os.path.exists(overrides_dir) and os.path.isdir(overrides_dir):
overrides = glob(overrides_dir + "/*.conf")
else:
overrides = []
# sequence things together
config.read(["./openwebrx.conf", "/etc/openwebrx/openwebrx.conf"] + overrides)
self.data_directory = config.get("core", "data_directory")
CoreConfig.checkDirectory(self.data_directory, "data_directory")
self.temporary_directory = config.get("core", "temporary_directory")
CoreConfig.checkDirectory(self.temporary_directory, "temporary_directory")
self.web_port = config.getint("web", "port")
self.aprs_symbols_path = config.get("aprs", "symbols_path")
@staticmethod
def checkDirectory(dir, key):
if not os.path.exists(dir):
raise ConfigError(key, "{dir} doesn't exist".format(dir=dir))
if not os.path.isdir(dir):
raise ConfigError(key, "{dir} is not a directory".format(dir=dir))
if not os.access(dir, os.W_OK):
raise ConfigError(key, "{dir} is not writable".format(dir=dir))
def get_web_port(self):
return self.web_port
def get_data_directory(self):
return self.data_directory
def get_temporary_directory(self):
return self.temporary_directory
def get_aprs_symbols_path(self):
return self.aprs_symbols_path
class Config:
sharedConfig = None
currentVersion = 3
migrators = {
1: ConfigMigratorVersion1(),
2: ConfigMigratorVersion2(),
}
@staticmethod
def _loadPythonFile(file):
spec = importlib.util.spec_from_file_location("config_webrx", file)
cfg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cfg)
pm = PropertyLayer()
for name, value in cfg.__dict__.items():
if name.startswith("__"):
continue
pm[name] = value
return pm
@staticmethod
def _loadJsonFile(file):
with open(file, "r") as f:
pm = PropertyLayer()
for k, v in json.load(f).items():
pm[k] = v
return pm
@staticmethod
def _getSettingsFile():
coreConfig = CoreConfig()
return "{data_directory}/settings.json".format(data_directory=coreConfig.get_data_directory())
@staticmethod
def _loadConfig():
for file in [Config._getSettingsFile(), "/etc/openwebrx/config_webrx.py", "./config_webrx.py"]:
try:
if file.endswith(".py"):
return Config._loadPythonFile(file)
elif file.endswith(".json"):
return Config._loadJsonFile(file)
else:
logger.warning("unsupported file type: %s", file)
except FileNotFoundError:
pass
raise ConfigNotFoundException("no usable config found! please make sure you have a valid configuration file!")
super().__init__()
self.storableConfig = DynamicConfig()
layers = [
self.storableConfig,
ClassicConfig(),
defaultConfig,
]
for i, l in enumerate(layers):
self.addLayer(i, l)
@staticmethod
def get():
if Config.sharedConfig is None:
Config.sharedConfig = Config._migrate(Config._loadConfig())
Config.sharedConfig = Config()
return Config.sharedConfig
@staticmethod
def store():
with open(Config._getSettingsFile(), "w") as file:
json.dump(Config.get().__dict__(), file, indent=4)
def store(self):
self.storableConfig.store()
@staticmethod
def validateConfig():
@ -131,14 +34,6 @@ class Config:
# just basic loading verification
Config.get()
@staticmethod
def _migrate(config):
version = config["version"] if "version" in config else 1
if version == Config.currentVersion:
return config
logger.debug("migrating config from version %i", version)
migrators = [Config.migrators[i] for i in range(version, Config.currentVersion)]
for migrator in migrators:
config = migrator.migrate(config)
return config
def __setitem__(self, key, value):
# in the config, all writes go to the json layer
return self.storableConfig.__setitem__(key, value)

30
owrx/config/classic.py Normal file
View File

@ -0,0 +1,30 @@
from owrx.property import PropertyReadOnly, PropertyLayer
from owrx.config.migration import Migrator
import importlib.util
class ClassicConfig(PropertyReadOnly):
def __init__(self):
pm = ClassicConfig._loadConfig()
Migrator.migrate(pm)
super().__init__(pm)
@staticmethod
def _loadConfig():
for file in ["/etc/openwebrx/config_webrx.py", "./config_webrx.py"]:
try:
return ClassicConfig._loadPythonFile(file)
except FileNotFoundError:
pass
@staticmethod
def _loadPythonFile(file):
spec = importlib.util.spec_from_file_location("config_webrx", file)
cfg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cfg)
pm = PropertyLayer()
for name, value in cfg.__dict__.items():
if name.startswith("__"):
continue
pm[name] = value
return pm

59
owrx/config/core.py Normal file
View File

@ -0,0 +1,59 @@
from owrx.config import ConfigError
from configparser import ConfigParser
import os
from glob import glob
class CoreConfig(object):
defaults = {
"core": {
"data_directory": "/var/lib/openwebrx",
"temporary_directory": "/tmp",
},
"web": {
"port": 8073,
},
"aprs": {
"symbols_path": "/usr/share/aprs-symbols/png"
}
}
def __init__(self):
config = ConfigParser()
# set up config defaults
config.read_dict(CoreConfig.defaults)
# check for overrides
overrides_dir = "/etc/openwebrx/openwebrx.conf.d"
if os.path.exists(overrides_dir) and os.path.isdir(overrides_dir):
overrides = glob(overrides_dir + "/*.conf")
else:
overrides = []
# sequence things together
config.read(["./openwebrx.conf", "/etc/openwebrx/openwebrx.conf"] + overrides)
self.data_directory = config.get("core", "data_directory")
CoreConfig.checkDirectory(self.data_directory, "data_directory")
self.temporary_directory = config.get("core", "temporary_directory")
CoreConfig.checkDirectory(self.temporary_directory, "temporary_directory")
self.web_port = config.getint("web", "port")
self.aprs_symbols_path = config.get("aprs", "symbols_path")
@staticmethod
def checkDirectory(dir, key):
if not os.path.exists(dir):
raise ConfigError(key, "{dir} doesn't exist".format(dir=dir))
if not os.path.isdir(dir):
raise ConfigError(key, "{dir} is not a directory".format(dir=dir))
if not os.access(dir, os.W_OK):
raise ConfigError(key, "{dir} is not writable".format(dir=dir))
def get_web_port(self):
return self.web_port
def get_data_directory(self):
return self.data_directory
def get_temporary_directory(self):
return self.temporary_directory
def get_aprs_symbols_path(self):
return self.aprs_symbols_path

323
owrx/config/defaults.py Normal file
View File

@ -0,0 +1,323 @@
from owrx.property import PropertyLayer
defaultConfig = PropertyLayer(
version=3,
max_clients=20,
receiver_name="[Callsign]",
receiver_location="Budapest, Hungary",
receiver_asl=200,
receiver_admin="example@example.com",
receiver_gps={"lat": 47.0, "lon": 19.0},
photo_title="Panorama of Budapest from Schönherz Zoltán Dormitory",
photo_desc="",
fft_fps=9,
fft_size=4096,
fft_voverlap_factor=0.3,
audio_compression="adpcm",
fft_compression="adpcm",
wfm_deemphasis_tau=50e-6,
digimodes_enable=True,
digimodes_fft_size=2048,
digital_voice_unvoiced_quality=1,
digital_voice_dmr_id_lookup=True,
# sdrs=...
waterfall_colors=[
0x30123B,
0x311542,
0x33184A,
0x341B51,
0x351E58,
0x36215F,
0x372466,
0x38266C,
0x392973,
0x3A2C79,
0x3B2F80,
0x3C3286,
0x3D358B,
0x3E3891,
0x3E3A97,
0x3F3D9C,
0x4040A2,
0x4043A7,
0x4146AC,
0x4248B1,
0x424BB6,
0x434EBA,
0x4351BF,
0x4453C3,
0x4456C7,
0x4559CB,
0x455BCF,
0x455ED3,
0x4561D7,
0x4663DA,
0x4666DD,
0x4669E1,
0x466BE4,
0x466EE7,
0x4671E9,
0x4673EC,
0x4676EE,
0x4678F1,
0x467BF3,
0x467DF5,
0x4680F7,
0x4682F9,
0x4685FA,
0x4587FC,
0x458AFD,
0x448CFE,
0x448FFE,
0x4391FF,
0x4294FF,
0x4196FF,
0x3F99FF,
0x3E9BFF,
0x3D9EFE,
0x3BA1FD,
0x3AA3FD,
0x38A6FB,
0x36A8FA,
0x35ABF9,
0x33ADF7,
0x31B0F6,
0x2FB2F4,
0x2DB5F2,
0x2CB7F0,
0x2AB9EE,
0x28BCEC,
0x26BEEA,
0x25C0E7,
0x23C3E5,
0x21C5E2,
0x20C7E0,
0x1FC9DD,
0x1DCCDB,
0x1CCED8,
0x1BD0D5,
0x1AD2D3,
0x19D4D0,
0x18D6CD,
0x18D8CB,
0x18DAC8,
0x17DBC5,
0x17DDC3,
0x17DFC0,
0x18E0BE,
0x18E2BB,
0x19E3B9,
0x1AE5B7,
0x1BE6B4,
0x1DE8B2,
0x1EE9AF,
0x20EAAD,
0x22ECAA,
0x24EDA7,
0x27EEA4,
0x29EFA1,
0x2CF09E,
0x2FF19B,
0x32F298,
0x35F394,
0x38F491,
0x3CF58E,
0x3FF68B,
0x43F787,
0x46F884,
0x4AF980,
0x4EFA7D,
0x51FA79,
0x55FB76,
0x59FC73,
0x5DFC6F,
0x61FD6C,
0x65FD69,
0x69FE65,
0x6DFE62,
0x71FE5F,
0x75FF5C,
0x79FF59,
0x7DFF56,
0x80FF53,
0x84FF50,
0x88FF4E,
0x8BFF4B,
0x8FFF49,
0x92FF46,
0x96FF44,
0x99FF42,
0x9CFE40,
0x9FFE3E,
0xA2FD3D,
0xA4FD3B,
0xA7FC3A,
0xAAFC39,
0xACFB38,
0xAFFA37,
0xB1F936,
0xB4F835,
0xB7F835,
0xB9F634,
0xBCF534,
0xBFF434,
0xC1F334,
0xC4F233,
0xC6F033,
0xC9EF34,
0xCBEE34,
0xCEEC34,
0xD0EB34,
0xD2E934,
0xD5E835,
0xD7E635,
0xD9E435,
0xDBE236,
0xDDE136,
0xE0DF37,
0xE2DD37,
0xE4DB38,
0xE6D938,
0xE7D738,
0xE9D539,
0xEBD339,
0xEDD139,
0xEECF3A,
0xF0CD3A,
0xF1CB3A,
0xF3C93A,
0xF4C73A,
0xF5C53A,
0xF7C33A,
0xF8C13A,
0xF9BF39,
0xFABD39,
0xFABA38,
0xFBB838,
0xFCB637,
0xFCB436,
0xFDB135,
0xFDAF35,
0xFEAC34,
0xFEA933,
0xFEA732,
0xFEA431,
0xFFA12F,
0xFF9E2E,
0xFF9C2D,
0xFF992C,
0xFE962B,
0xFE932A,
0xFE9028,
0xFE8D27,
0xFD8A26,
0xFD8724,
0xFC8423,
0xFC8122,
0xFB7E20,
0xFB7B1F,
0xFA781E,
0xF9751C,
0xF8721B,
0xF86F1A,
0xF76C19,
0xF66917,
0xF56616,
0xF46315,
0xF36014,
0xF25D13,
0xF05B11,
0xEF5810,
0xEE550F,
0xED530E,
0xEB500E,
0xEA4E0D,
0xE94B0C,
0xE7490B,
0xE6470A,
0xE4450A,
0xE34209,
0xE14009,
0xDF3E08,
0xDE3C07,
0xDC3A07,
0xDA3806,
0xD83606,
0xD63405,
0xD43205,
0xD23105,
0xD02F04,
0xCE2D04,
0xCC2B03,
0xCA2903,
0xC82803,
0xC62602,
0xC32402,
0xC12302,
0xBF2102,
0xBC1F01,
0xBA1E01,
0xB71C01,
0xB41B01,
0xB21901,
0xAF1801,
0xAC1601,
0xAA1501,
0xA71401,
0xA41201,
0xA11101,
0x9E1001,
0x9B0F01,
0x980D01,
0x950C01,
0x920B01,
0x8E0A01,
0x8B0901,
0x880801,
0x850701,
0x810602,
0x7E0502,
0x7A0402,
],
waterfall_min_level=-88,
waterfall_max_level=-20,
waterfall_auto_level_margin={"min": 3, "max": 10, "min_range": 50},
frequency_display_precision=4,
squelch_auto_margin=10,
# TODO: deprecated. remove from code.
csdr_dynamic_bufsize=False,
# TODO: deprecated. remove from code.
csdr_print_bufsizes=False,
# TODO: deprecated. remove from code.
csdr_through=False,
nmux_memory=50,
google_maps_api_key="",
map_position_retention_time=2 * 60 * 60,
decoding_queue_workers=2,
decoding_queue_length=10,
wsjt_decoding_depth=3,
wsjt_decoding_depths={"jt65": 1},
fst4_enabled_intervals=[15, 30],
fst4w_enabled_intervals=[120, 300],
q65_enabled_combinations=["A30", "E120", "C60"],
js8_enabled_profiles=["normal", "slow"],
js8_decoding_depth=3,
services_enabled=False,
services_decoders=["ft8", "ft4", "wspr", "packet"],
aprs_callsign="N0CALL",
aprs_igate_enabled=False,
aprs_igate_server="euro.aprs2.net",
aprs_igate_password="",
aprs_igate_beacon=False,
aprs_igate_symbol="R&",
aprs_igate_comment="OpenWebRX APRS gateway",
aprs_igate_height=None,
aprs_igate_gain=None,
aprs_igate_dir=None,
pskreporter_enabled=False,
pskreporter_callsign="N0CALL",
pskreporter_antenna_information=None,
wsprnet_enabled=False,
wsprnet_callsign="N0CALL",
).readonly()

25
owrx/config/dynamic.py Normal file
View File

@ -0,0 +1,25 @@
from owrx.config.core import CoreConfig
from owrx.config.migration import Migrator
from owrx.property import PropertyLayer
import json
class DynamicConfig(PropertyLayer):
def __init__(self):
super().__init__()
try:
with open(DynamicConfig._getSettingsFile(), "r") as f:
for k, v in json.load(f).items():
self[k] = v
except FileNotFoundError:
pass
Migrator.migrate(self)
@staticmethod
def _getSettingsFile():
coreConfig = CoreConfig()
return "{data_directory}/settings.json".format(data_directory=coreConfig.get_data_directory())
def store(self):
with open(DynamicConfig._getSettingsFile(), "w") as file:
json.dump(self.__dict__(), file, indent=4)

4
owrx/config/error.py
View File

@ -1,7 +1,3 @@
class ConfigNotFoundException(Exception):
pass
class ConfigError(Exception):
def __init__(self, key, message):
super().__init__("Configuration Error (key: {0}): {1}".format(key, message))

27
owrx/config/migration.py
View File

@ -1,5 +1,9 @@
from abc import ABC, abstractmethod
import logging
logger = logging.getLogger(__name__)
class ConfigMigrator(ABC):
@abstractmethod
@ -26,11 +30,30 @@ class ConfigMigratorVersion1(ConfigMigrator):
self.renameKey(config, "wsjt_queue_length", "decoding_queue_length")
config["version"] = 2
return config
class ConfigMigratorVersion2(ConfigMigrator):
def migrate(self, config):
if "waterfall_colors" in config and any(v > 0xFFFFFF for v in config["waterfall_colors"]):
config["waterfall_colors"] = [v >> 8 for v in config["waterfall_colors"]]
return config
config["version"] = 3
class Migrator(object):
currentVersion = 3
migrators = {
1: ConfigMigratorVersion1(),
2: ConfigMigratorVersion2(),
}
@staticmethod
def migrate(config):
version = config["version"] if "version" in config else 1
if version == Migrator.currentVersion:
return config
logger.debug("migrating config from version %i", version)
migrators = [Migrator.migrators[i] for i in range(version, Migrator.currentVersion)]
for migrator in migrators:
migrator.migrate(config)

2
owrx/controllers/assets.py
View File

@ -1,5 +1,5 @@
from . import Controller
from owrx.config import CoreConfig
from owrx.config.core import CoreConfig
from datetime import datetime, timezone
import mimetypes
import os

2
owrx/controllers/imageupload.py
View File

@ -1,6 +1,6 @@
from owrx.controllers.assets import AssetsController
from owrx.controllers.admin import AuthorizationMixin
from owrx.config import CoreConfig
from owrx.config.core import CoreConfig
import uuid
import json

5
owrx/controllers/settings.py
View File

@ -1,6 +1,7 @@
from owrx.controllers.template import WebpageController
from owrx.controllers.admin import AuthorizationMixin
from owrx.config import Config, CoreConfig
from owrx.config.core import CoreConfig
from owrx.config import Config
from urllib.parse import parse_qs
from owrx.form import (
TextInput,
@ -408,5 +409,5 @@ class GeneralSettingsController(AuthorizationMixin, WebpageController):
del config[k]
else:
config[k] = v
Config.store()
config.store()
self.send_redirect("/generalsettings")

2
owrx/dsp.py
View File

@ -7,7 +7,7 @@ from owrx.source import SdrSource, SdrSourceEventClient
from owrx.property import PropertyStack, PropertyLayer, PropertyValidator
from owrx.property.validators import OrValidator, RegexValidator, BoolValidator
from owrx.modes import Modes
from owrx.config import CoreConfig
from owrx.config.core import CoreConfig
from csdr import csdr
import threading
import re

2
owrx/feature.py
View File

@ -4,7 +4,7 @@ from operator import and_
import re
from distutils.version import LooseVersion
import inspect
from owrx.config import CoreConfig
from owrx.config.core import CoreConfig
import shlex
import os
from datetime import datetime, timedelta

3
owrx/fft.py
View File

@ -1,4 +1,5 @@
from owrx.config import Config, CoreConfig
from owrx.config.core import CoreConfig
from owrx.config import Config
from csdr import csdr
import threading
from owrx.source import SdrSource, SdrSourceEventClient

8
owrx/property/__init__.py
View File

@ -98,9 +98,10 @@ class PropertyManager(ABC):
class PropertyLayer(PropertyManager):
def __init__(self):
def __init__(self, **kwargs):
super().__init__()
self.properties = {}
# copy, don't re-use
self.properties = {k: v for k, v in kwargs.items()}
def __contains__(self, name):
return name in self.properties
@ -311,7 +312,8 @@ class PropertyStack(PropertyManager):
def __delitem__(self, key):
for layer in self.layers:
layer["props"].__delitem__(key)
if layer["props"].__contains__(key):
layer["props"].__delitem__(key)
def keys(self):
return set([key for l in self.layers for key in l["props"].keys()])

3
owrx/service/__init__.py
View File

@ -6,7 +6,8 @@ from csdr.csdr import dsp, output
from owrx.wsjt import WsjtParser
from owrx.aprs import AprsParser
from owrx.js8 import Js8Parser
from owrx.config import Config, CoreConfig
from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.source.resampler import Resampler
from owrx.property import PropertyLayer
from js8py import Js8Frame

2
owrx/users.py
View File

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from owrx.config import CoreConfig
from owrx.config.core import CoreConfig
from datetime import datetime, timezone
import json
import hashlib

9
test/property/test_property_layer.py
View File

@ -4,6 +4,15 @@ from unittest.mock import Mock
class PropertyLayerTest(TestCase):
def testCreationWithKwArgs(self):
pm = PropertyLayer(testkey="value")
self.assertEqual(pm["testkey"], "value")
# this should be synonymous, so this is rather for illustration purposes
contents = {"testkey": "value"}
pm = PropertyLayer(**contents)
self.assertEqual(pm["testkey"], "value")
def testKeyIsset(self):
pm = PropertyLayer()
self.assertFalse("some_key" in pm)

10
test/property/test_property_stack.py
View File

@ -185,3 +185,13 @@ class PropertyStackTest(TestCase):
stack.replaceLayer(0, second_layer)
mock.method.assert_not_called()
def testWritesToExpectedLayer(self):
om = PropertyStack()
low_pm = PropertyLayer()
high_pm = PropertyLayer()
low_pm["testkey"] = "low value"
om.addLayer(1, low_pm)
om.addLayer(0, high_pm)
om["testkey"] = "new value"
self.assertEqual(low_pm["testkey"], "new value")