implement config layering

This commit is contained in:
Jakob Ketterl 2021-02-11 19:31:44 +01:00
parent e926611307
commit f23fa59ac3
20 changed files with 524 additions and 147 deletions

View File

@ -1,6 +1,7 @@
from http.server import HTTPServer from http.server import HTTPServer
from owrx.http import RequestHandler from owrx.http import RequestHandler
from owrx.config import Config, CoreConfig from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.feature import FeatureDetector from owrx.feature import FeatureDetector
from owrx.sdr import SdrService from owrx.sdr import SdrService
from socketserver import ThreadingMixIn from socketserver import ThreadingMixIn

View File

@ -1,5 +1,6 @@
from abc import ABC, ABCMeta, abstractmethod from abc import ABC, ABCMeta, abstractmethod
from owrx.config import Config, CoreConfig from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.metrics import Metrics, CounterMetric, DirectMetric from owrx.metrics import Metrics, CounterMetric, DirectMetric
import threading import threading
import wave import wave

View File

@ -1,129 +1,32 @@
from configparser import ConfigParser from owrx.property import PropertyStack
from owrx.property import PropertyLayer from owrx.config.error import ConfigError
import importlib.util from owrx.config.defaults import defaultConfig
import os from owrx.config.dynamic import DynamicConfig
import json from owrx.config.classic import ClassicConfig
from glob import glob
from owrx.config.error import ConfigError, ConfigNotFoundException
from owrx.config.migration import ConfigMigratorVersion1, ConfigMigratorVersion2
import logging
logger = logging.getLogger(__name__)
class CoreConfig(object): class Config(PropertyStack):
defaults = { sharedConfig = None
"core": {
"data_directory": "/var/lib/openwebrx",
"temporary_directory": "/tmp",
},
"web": {
"port": 8073,
},
"aprs": {
"symbols_path": "/usr/share/aprs-symbols/png"
}
}
def __init__(self): def __init__(self):
config = ConfigParser() super().__init__()
# set up config defaults self.storableConfig = DynamicConfig()
config.read_dict(CoreConfig.defaults) layers = [
# check for overrides self.storableConfig,
overrides_dir = "/etc/openwebrx/openwebrx.conf.d" ClassicConfig(),
if os.path.exists(overrides_dir) and os.path.isdir(overrides_dir): defaultConfig,
overrides = glob(overrides_dir + "/*.conf") ]
else: for i, l in enumerate(layers):
overrides = [] self.addLayer(i, l)
# sequence things together
config.read(["./openwebrx.conf", "/etc/openwebrx/openwebrx.conf"] + overrides)
self.data_directory = config.get("core", "data_directory")
CoreConfig.checkDirectory(self.data_directory, "data_directory")
self.temporary_directory = config.get("core", "temporary_directory")
CoreConfig.checkDirectory(self.temporary_directory, "temporary_directory")
self.web_port = config.getint("web", "port")
self.aprs_symbols_path = config.get("aprs", "symbols_path")
@staticmethod
def checkDirectory(dir, key):
if not os.path.exists(dir):
raise ConfigError(key, "{dir} doesn't exist".format(dir=dir))
if not os.path.isdir(dir):
raise ConfigError(key, "{dir} is not a directory".format(dir=dir))
if not os.access(dir, os.W_OK):
raise ConfigError(key, "{dir} is not writable".format(dir=dir))
def get_web_port(self):
return self.web_port
def get_data_directory(self):
return self.data_directory
def get_temporary_directory(self):
return self.temporary_directory
def get_aprs_symbols_path(self):
return self.aprs_symbols_path
class Config:
sharedConfig = None
currentVersion = 3
migrators = {
1: ConfigMigratorVersion1(),
2: ConfigMigratorVersion2(),
}
@staticmethod
def _loadPythonFile(file):
spec = importlib.util.spec_from_file_location("config_webrx", file)
cfg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cfg)
pm = PropertyLayer()
for name, value in cfg.__dict__.items():
if name.startswith("__"):
continue
pm[name] = value
return pm
@staticmethod
def _loadJsonFile(file):
with open(file, "r") as f:
pm = PropertyLayer()
for k, v in json.load(f).items():
pm[k] = v
return pm
@staticmethod
def _getSettingsFile():
coreConfig = CoreConfig()
return "{data_directory}/settings.json".format(data_directory=coreConfig.get_data_directory())
@staticmethod
def _loadConfig():
for file in [Config._getSettingsFile(), "/etc/openwebrx/config_webrx.py", "./config_webrx.py"]:
try:
if file.endswith(".py"):
return Config._loadPythonFile(file)
elif file.endswith(".json"):
return Config._loadJsonFile(file)
else:
logger.warning("unsupported file type: %s", file)
except FileNotFoundError:
pass
raise ConfigNotFoundException("no usable config found! please make sure you have a valid configuration file!")
@staticmethod @staticmethod
def get(): def get():
if Config.sharedConfig is None: if Config.sharedConfig is None:
Config.sharedConfig = Config._migrate(Config._loadConfig()) Config.sharedConfig = Config()
return Config.sharedConfig return Config.sharedConfig
@staticmethod def store(self):
def store(): self.storableConfig.store()
with open(Config._getSettingsFile(), "w") as file:
json.dump(Config.get().__dict__(), file, indent=4)
@staticmethod @staticmethod
def validateConfig(): def validateConfig():
@ -131,14 +34,6 @@ class Config:
# just basic loading verification # just basic loading verification
Config.get() Config.get()
@staticmethod def __setitem__(self, key, value):
def _migrate(config): # in the config, all writes go to the json layer
version = config["version"] if "version" in config else 1 return self.storableConfig.__setitem__(key, value)
if version == Config.currentVersion:
return config
logger.debug("migrating config from version %i", version)
migrators = [Config.migrators[i] for i in range(version, Config.currentVersion)]
for migrator in migrators:
config = migrator.migrate(config)
return config

30
owrx/config/classic.py Normal file
View File

@ -0,0 +1,30 @@
from owrx.property import PropertyReadOnly, PropertyLayer
from owrx.config.migration import Migrator
import importlib.util
class ClassicConfig(PropertyReadOnly):
def __init__(self):
pm = ClassicConfig._loadConfig()
Migrator.migrate(pm)
super().__init__(pm)
@staticmethod
def _loadConfig():
for file in ["/etc/openwebrx/config_webrx.py", "./config_webrx.py"]:
try:
return ClassicConfig._loadPythonFile(file)
except FileNotFoundError:
pass
@staticmethod
def _loadPythonFile(file):
spec = importlib.util.spec_from_file_location("config_webrx", file)
cfg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cfg)
pm = PropertyLayer()
for name, value in cfg.__dict__.items():
if name.startswith("__"):
continue
pm[name] = value
return pm

59
owrx/config/core.py Normal file
View File

@ -0,0 +1,59 @@
from owrx.config import ConfigError
from configparser import ConfigParser
import os
from glob import glob
class CoreConfig(object):
defaults = {
"core": {
"data_directory": "/var/lib/openwebrx",
"temporary_directory": "/tmp",
},
"web": {
"port": 8073,
},
"aprs": {
"symbols_path": "/usr/share/aprs-symbols/png"
}
}
def __init__(self):
config = ConfigParser()
# set up config defaults
config.read_dict(CoreConfig.defaults)
# check for overrides
overrides_dir = "/etc/openwebrx/openwebrx.conf.d"
if os.path.exists(overrides_dir) and os.path.isdir(overrides_dir):
overrides = glob(overrides_dir + "/*.conf")
else:
overrides = []
# sequence things together
config.read(["./openwebrx.conf", "/etc/openwebrx/openwebrx.conf"] + overrides)
self.data_directory = config.get("core", "data_directory")
CoreConfig.checkDirectory(self.data_directory, "data_directory")
self.temporary_directory = config.get("core", "temporary_directory")
CoreConfig.checkDirectory(self.temporary_directory, "temporary_directory")
self.web_port = config.getint("web", "port")
self.aprs_symbols_path = config.get("aprs", "symbols_path")
@staticmethod
def checkDirectory(dir, key):
if not os.path.exists(dir):
raise ConfigError(key, "{dir} doesn't exist".format(dir=dir))
if not os.path.isdir(dir):
raise ConfigError(key, "{dir} is not a directory".format(dir=dir))
if not os.access(dir, os.W_OK):
raise ConfigError(key, "{dir} is not writable".format(dir=dir))
def get_web_port(self):
return self.web_port
def get_data_directory(self):
return self.data_directory
def get_temporary_directory(self):
return self.temporary_directory
def get_aprs_symbols_path(self):
return self.aprs_symbols_path

323
owrx/config/defaults.py Normal file
View File

@ -0,0 +1,323 @@
from owrx.property import PropertyLayer
defaultConfig = PropertyLayer(
version=3,
max_clients=20,
receiver_name="[Callsign]",
receiver_location="Budapest, Hungary",
receiver_asl=200,
receiver_admin="example@example.com",
receiver_gps={"lat": 47.0, "lon": 19.0},
photo_title="Panorama of Budapest from Schönherz Zoltán Dormitory",
photo_desc="",
fft_fps=9,
fft_size=4096,
fft_voverlap_factor=0.3,
audio_compression="adpcm",
fft_compression="adpcm",
wfm_deemphasis_tau=50e-6,
digimodes_enable=True,
digimodes_fft_size=2048,
digital_voice_unvoiced_quality=1,
digital_voice_dmr_id_lookup=True,
# sdrs=...
waterfall_colors=[
0x30123B,
0x311542,
0x33184A,
0x341B51,
0x351E58,
0x36215F,
0x372466,
0x38266C,
0x392973,
0x3A2C79,
0x3B2F80,
0x3C3286,
0x3D358B,
0x3E3891,
0x3E3A97,
0x3F3D9C,
0x4040A2,
0x4043A7,
0x4146AC,
0x4248B1,
0x424BB6,
0x434EBA,
0x4351BF,
0x4453C3,
0x4456C7,
0x4559CB,
0x455BCF,
0x455ED3,
0x4561D7,
0x4663DA,
0x4666DD,
0x4669E1,
0x466BE4,
0x466EE7,
0x4671E9,
0x4673EC,
0x4676EE,
0x4678F1,
0x467BF3,
0x467DF5,
0x4680F7,
0x4682F9,
0x4685FA,
0x4587FC,
0x458AFD,
0x448CFE,
0x448FFE,
0x4391FF,
0x4294FF,
0x4196FF,
0x3F99FF,
0x3E9BFF,
0x3D9EFE,
0x3BA1FD,
0x3AA3FD,
0x38A6FB,
0x36A8FA,
0x35ABF9,
0x33ADF7,
0x31B0F6,
0x2FB2F4,
0x2DB5F2,
0x2CB7F0,
0x2AB9EE,
0x28BCEC,
0x26BEEA,
0x25C0E7,
0x23C3E5,
0x21C5E2,
0x20C7E0,
0x1FC9DD,
0x1DCCDB,
0x1CCED8,
0x1BD0D5,
0x1AD2D3,
0x19D4D0,
0x18D6CD,
0x18D8CB,
0x18DAC8,
0x17DBC5,
0x17DDC3,
0x17DFC0,
0x18E0BE,
0x18E2BB,
0x19E3B9,
0x1AE5B7,
0x1BE6B4,
0x1DE8B2,
0x1EE9AF,
0x20EAAD,
0x22ECAA,
0x24EDA7,
0x27EEA4,
0x29EFA1,
0x2CF09E,
0x2FF19B,
0x32F298,
0x35F394,
0x38F491,
0x3CF58E,
0x3FF68B,
0x43F787,
0x46F884,
0x4AF980,
0x4EFA7D,
0x51FA79,
0x55FB76,
0x59FC73,
0x5DFC6F,
0x61FD6C,
0x65FD69,
0x69FE65,
0x6DFE62,
0x71FE5F,
0x75FF5C,
0x79FF59,
0x7DFF56,
0x80FF53,
0x84FF50,
0x88FF4E,
0x8BFF4B,
0x8FFF49,
0x92FF46,
0x96FF44,
0x99FF42,
0x9CFE40,
0x9FFE3E,
0xA2FD3D,
0xA4FD3B,
0xA7FC3A,
0xAAFC39,
0xACFB38,
0xAFFA37,
0xB1F936,
0xB4F835,
0xB7F835,
0xB9F634,
0xBCF534,
0xBFF434,
0xC1F334,
0xC4F233,
0xC6F033,
0xC9EF34,
0xCBEE34,
0xCEEC34,
0xD0EB34,
0xD2E934,
0xD5E835,
0xD7E635,
0xD9E435,
0xDBE236,
0xDDE136,
0xE0DF37,
0xE2DD37,
0xE4DB38,
0xE6D938,
0xE7D738,
0xE9D539,
0xEBD339,
0xEDD139,
0xEECF3A,
0xF0CD3A,
0xF1CB3A,
0xF3C93A,
0xF4C73A,
0xF5C53A,
0xF7C33A,
0xF8C13A,
0xF9BF39,
0xFABD39,
0xFABA38,
0xFBB838,
0xFCB637,
0xFCB436,
0xFDB135,
0xFDAF35,
0xFEAC34,
0xFEA933,
0xFEA732,
0xFEA431,
0xFFA12F,
0xFF9E2E,
0xFF9C2D,
0xFF992C,
0xFE962B,
0xFE932A,
0xFE9028,
0xFE8D27,
0xFD8A26,
0xFD8724,
0xFC8423,
0xFC8122,
0xFB7E20,
0xFB7B1F,
0xFA781E,
0xF9751C,
0xF8721B,
0xF86F1A,
0xF76C19,
0xF66917,
0xF56616,
0xF46315,
0xF36014,
0xF25D13,
0xF05B11,
0xEF5810,
0xEE550F,
0xED530E,
0xEB500E,
0xEA4E0D,
0xE94B0C,
0xE7490B,
0xE6470A,
0xE4450A,
0xE34209,
0xE14009,
0xDF3E08,
0xDE3C07,
0xDC3A07,
0xDA3806,
0xD83606,
0xD63405,
0xD43205,
0xD23105,
0xD02F04,
0xCE2D04,
0xCC2B03,
0xCA2903,
0xC82803,
0xC62602,
0xC32402,
0xC12302,
0xBF2102,
0xBC1F01,
0xBA1E01,
0xB71C01,
0xB41B01,
0xB21901,
0xAF1801,
0xAC1601,
0xAA1501,
0xA71401,
0xA41201,
0xA11101,
0x9E1001,
0x9B0F01,
0x980D01,
0x950C01,
0x920B01,
0x8E0A01,
0x8B0901,
0x880801,
0x850701,
0x810602,
0x7E0502,
0x7A0402,
],
waterfall_min_level=-88,
waterfall_max_level=-20,
waterfall_auto_level_margin={"min": 3, "max": 10, "min_range": 50},
frequency_display_precision=4,
squelch_auto_margin=10,
# TODO: deprecated. remove from code.
csdr_dynamic_bufsize=False,
# TODO: deprecated. remove from code.
csdr_print_bufsizes=False,
# TODO: deprecated. remove from code.
csdr_through=False,
nmux_memory=50,
google_maps_api_key="",
map_position_retention_time=2 * 60 * 60,
decoding_queue_workers=2,
decoding_queue_length=10,
wsjt_decoding_depth=3,
wsjt_decoding_depths={"jt65": 1},
fst4_enabled_intervals=[15, 30],
fst4w_enabled_intervals=[120, 300],
q65_enabled_combinations=["A30", "E120", "C60"],
js8_enabled_profiles=["normal", "slow"],
js8_decoding_depth=3,
services_enabled=False,
services_decoders=["ft8", "ft4", "wspr", "packet"],
aprs_callsign="N0CALL",
aprs_igate_enabled=False,
aprs_igate_server="euro.aprs2.net",
aprs_igate_password="",
aprs_igate_beacon=False,
aprs_igate_symbol="R&",
aprs_igate_comment="OpenWebRX APRS gateway",
aprs_igate_height=None,
aprs_igate_gain=None,
aprs_igate_dir=None,
pskreporter_enabled=False,
pskreporter_callsign="N0CALL",
pskreporter_antenna_information=None,
wsprnet_enabled=False,
wsprnet_callsign="N0CALL",
).readonly()

25
owrx/config/dynamic.py Normal file
View File

@ -0,0 +1,25 @@
from owrx.config.core import CoreConfig
from owrx.config.migration import Migrator
from owrx.property import PropertyLayer
import json
class DynamicConfig(PropertyLayer):
def __init__(self):
super().__init__()
try:
with open(DynamicConfig._getSettingsFile(), "r") as f:
for k, v in json.load(f).items():
self[k] = v
except FileNotFoundError:
pass
Migrator.migrate(self)
@staticmethod
def _getSettingsFile():
coreConfig = CoreConfig()
return "{data_directory}/settings.json".format(data_directory=coreConfig.get_data_directory())
def store(self):
with open(DynamicConfig._getSettingsFile(), "w") as file:
json.dump(self.__dict__(), file, indent=4)

View File

@ -1,7 +1,3 @@
class ConfigNotFoundException(Exception):
pass
class ConfigError(Exception): class ConfigError(Exception):
def __init__(self, key, message): def __init__(self, key, message):
super().__init__("Configuration Error (key: {0}): {1}".format(key, message)) super().__init__("Configuration Error (key: {0}): {1}".format(key, message))

View File

@ -1,5 +1,9 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import logging
logger = logging.getLogger(__name__)
class ConfigMigrator(ABC): class ConfigMigrator(ABC):
@abstractmethod @abstractmethod
@ -26,11 +30,30 @@ class ConfigMigratorVersion1(ConfigMigrator):
self.renameKey(config, "wsjt_queue_length", "decoding_queue_length") self.renameKey(config, "wsjt_queue_length", "decoding_queue_length")
config["version"] = 2 config["version"] = 2
return config
class ConfigMigratorVersion2(ConfigMigrator): class ConfigMigratorVersion2(ConfigMigrator):
def migrate(self, config): def migrate(self, config):
if "waterfall_colors" in config and any(v > 0xFFFFFF for v in config["waterfall_colors"]): if "waterfall_colors" in config and any(v > 0xFFFFFF for v in config["waterfall_colors"]):
config["waterfall_colors"] = [v >> 8 for v in config["waterfall_colors"]] config["waterfall_colors"] = [v >> 8 for v in config["waterfall_colors"]]
return config
config["version"] = 3
class Migrator(object):
currentVersion = 3
migrators = {
1: ConfigMigratorVersion1(),
2: ConfigMigratorVersion2(),
}
@staticmethod
def migrate(config):
version = config["version"] if "version" in config else 1
if version == Migrator.currentVersion:
return config
logger.debug("migrating config from version %i", version)
migrators = [Migrator.migrators[i] for i in range(version, Migrator.currentVersion)]
for migrator in migrators:
migrator.migrate(config)

View File

@ -1,5 +1,5 @@
from . import Controller from . import Controller
from owrx.config import CoreConfig from owrx.config.core import CoreConfig
from datetime import datetime, timezone from datetime import datetime, timezone
import mimetypes import mimetypes
import os import os

View File

@ -1,6 +1,6 @@
from owrx.controllers.assets import AssetsController from owrx.controllers.assets import AssetsController
from owrx.controllers.admin import AuthorizationMixin from owrx.controllers.admin import AuthorizationMixin
from owrx.config import CoreConfig from owrx.config.core import CoreConfig
import uuid import uuid
import json import json

View File

@ -1,6 +1,7 @@
from owrx.controllers.template import WebpageController from owrx.controllers.template import WebpageController
from owrx.controllers.admin import AuthorizationMixin from owrx.controllers.admin import AuthorizationMixin
from owrx.config import Config, CoreConfig from owrx.config.core import CoreConfig
from owrx.config import Config
from urllib.parse import parse_qs from urllib.parse import parse_qs
from owrx.form import ( from owrx.form import (
TextInput, TextInput,
@ -408,5 +409,5 @@ class GeneralSettingsController(AuthorizationMixin, WebpageController):
del config[k] del config[k]
else: else:
config[k] = v config[k] = v
Config.store() config.store()
self.send_redirect("/generalsettings") self.send_redirect("/generalsettings")

View File

@ -7,7 +7,7 @@ from owrx.source import SdrSource, SdrSourceEventClient
from owrx.property import PropertyStack, PropertyLayer, PropertyValidator from owrx.property import PropertyStack, PropertyLayer, PropertyValidator
from owrx.property.validators import OrValidator, RegexValidator, BoolValidator from owrx.property.validators import OrValidator, RegexValidator, BoolValidator
from owrx.modes import Modes from owrx.modes import Modes
from owrx.config import CoreConfig from owrx.config.core import CoreConfig
from csdr import csdr from csdr import csdr
import threading import threading
import re import re

View File

@ -4,7 +4,7 @@ from operator import and_
import re import re
from distutils.version import LooseVersion from distutils.version import LooseVersion
import inspect import inspect
from owrx.config import CoreConfig from owrx.config.core import CoreConfig
import shlex import shlex
import os import os
from datetime import datetime, timedelta from datetime import datetime, timedelta

View File

@ -1,4 +1,5 @@
from owrx.config import Config, CoreConfig from owrx.config.core import CoreConfig
from owrx.config import Config
from csdr import csdr from csdr import csdr
import threading import threading
from owrx.source import SdrSource, SdrSourceEventClient from owrx.source import SdrSource, SdrSourceEventClient

View File

@ -98,9 +98,10 @@ class PropertyManager(ABC):
class PropertyLayer(PropertyManager): class PropertyLayer(PropertyManager):
def __init__(self): def __init__(self, **kwargs):
super().__init__() super().__init__()
self.properties = {} # copy, don't re-use
self.properties = {k: v for k, v in kwargs.items()}
def __contains__(self, name): def __contains__(self, name):
return name in self.properties return name in self.properties
@ -311,7 +312,8 @@ class PropertyStack(PropertyManager):
def __delitem__(self, key): def __delitem__(self, key):
for layer in self.layers: for layer in self.layers:
layer["props"].__delitem__(key) if layer["props"].__contains__(key):
layer["props"].__delitem__(key)
def keys(self): def keys(self):
return set([key for l in self.layers for key in l["props"].keys()]) return set([key for l in self.layers for key in l["props"].keys()])

View File

@ -6,7 +6,8 @@ from csdr.csdr import dsp, output
from owrx.wsjt import WsjtParser from owrx.wsjt import WsjtParser
from owrx.aprs import AprsParser from owrx.aprs import AprsParser
from owrx.js8 import Js8Parser from owrx.js8 import Js8Parser
from owrx.config import Config, CoreConfig from owrx.config.core import CoreConfig
from owrx.config import Config
from owrx.source.resampler import Resampler from owrx.source.resampler import Resampler
from owrx.property import PropertyLayer from owrx.property import PropertyLayer
from js8py import Js8Frame from js8py import Js8Frame

View File

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from owrx.config import CoreConfig from owrx.config.core import CoreConfig
from datetime import datetime, timezone from datetime import datetime, timezone
import json import json
import hashlib import hashlib

View File

@ -4,6 +4,15 @@ from unittest.mock import Mock
class PropertyLayerTest(TestCase): class PropertyLayerTest(TestCase):
def testCreationWithKwArgs(self):
pm = PropertyLayer(testkey="value")
self.assertEqual(pm["testkey"], "value")
# this should be synonymous, so this is rather for illustration purposes
contents = {"testkey": "value"}
pm = PropertyLayer(**contents)
self.assertEqual(pm["testkey"], "value")
def testKeyIsset(self): def testKeyIsset(self):
pm = PropertyLayer() pm = PropertyLayer()
self.assertFalse("some_key" in pm) self.assertFalse("some_key" in pm)

View File

@ -185,3 +185,13 @@ class PropertyStackTest(TestCase):
stack.replaceLayer(0, second_layer) stack.replaceLayer(0, second_layer)
mock.method.assert_not_called() mock.method.assert_not_called()
def testWritesToExpectedLayer(self):
om = PropertyStack()
low_pm = PropertyLayer()
high_pm = PropertyLayer()
low_pm["testkey"] = "low value"
om.addLayer(1, low_pm)
om.addLayer(0, high_pm)
om["testkey"] = "new value"
self.assertEqual(low_pm["testkey"], "new value")