From faedf2bfc7e9334f6a4f55682a16fc886193c210 Mon Sep 17 00:00:00 2001 From: Taryel Hlontsi Date: Sun, 16 Jul 2023 05:38:03 +0200 Subject: [PATCH] Add frequency plotting and representation values in binary format --- README.md | 8 +- pyproject.toml | 1 + src/data/__init__.py | 23 ++- src/data/controller-models.json | 174 ++++++++++++++---- src/data/controllers.json | 12 +- src/main/__init__.py | 3 +- src/modbus/simulator.py | 135 +++++++++++++- src/ui/controller_types.py | 20 +- src/ui/controllers.py | 61 ++++-- src/ui/static/style.css | 30 ++- src/ui/templates/base.html | 4 + .../controller_types/create-back.html | 27 --- src/ui/templates/controller_types/list.html | 13 +- src/ui/templates/controller_types/update.html | 11 +- src/ui/templates/controllers/index.html | 18 +- 15 files changed, 420 insertions(+), 120 deletions(-) delete mode 100644 src/ui/templates/controller_types/create-back.html diff --git a/README.md b/README.md index da5447e..4c345c4 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Setup & debug + ``pip list`` (pip show abibas, pip uninstall abibas, etc) At this stage the package is installed in editable mode, to run it in debug mode: - +, + ``abibas-debug`` or in normal mode: @@ -40,3 +40,9 @@ If in debug mode changes made to the Flask app will be applied automatically at for the changes in simulator though the tool has to be restarted. **main** module can be manipulated for debugging purposes as well + +Before releasing +================ ++ ``pip install twine`` ++ ``twine check dist/*`` ++ ``twine upload --repository-url https://test.pypi.org/legacy/ dist/*`` diff --git a/pyproject.toml b/pyproject.toml index 41bdb7c..51eec02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "Flask", "pyModbusTCP", "prettytable", + "plotext", ] version = "0.0.1" urls = {homepage = "https://gittar.crabdance.com/e-corp/abibas"} diff --git a/src/data/__init__.py b/src/data/__init__.py index 2108335..02eb0e3 100644 --- a/src/data/__init__.py +++ b/src/data/__init__.py @@ -34,8 +34,11 @@ def _get_controller_models_fname() -> str: def _get_reg_value(controller: dict, reg: dict) -> int: # get first value from type definition (as default) value = None - k, v = next(iter(reg['PossibleValues'][0].items()), None) - value = v + if 'PossibleValues' not in reg or not reg['PossibleValues']: + value = 0 + else: + k, v = next(iter(reg['PossibleValues'][0].items()), None) + value = v # if there is a file with overriden value, take that overrides = _get_overrides_fname(controller['Type'], controller['Port']) if os.path.isfile(overrides): @@ -51,8 +54,8 @@ def _get_reg_value(controller: dict, reg: dict) -> int: entry = [x for x in overriden if x['Name'] == reg['Name']] if entry: value = entry[0]['Value'] - # logging.debug( - # f'{reg["Name"]} value gotten: {value} from {overrides}') + logging.debug( + f'{reg["Name"]} value gotten: {value} from {overrides}') return value @@ -61,11 +64,6 @@ def _save_overrides(fname: str, overrides: list): json.dump(overrides, file, indent=2) -def get_version() -> str: - version = pkg_resources.get_distribution('abibas').version - return version - - def get_controller(port: int) -> dict: controllers = get_controllers() models = get_controller_types() @@ -164,3 +162,10 @@ def remove_controller(port: int) -> bool: logging.debug(f'Overrides file {overrides} not found, skip removing') return True + + +def get_info() -> dict: + distribution = pkg_resources.get_distribution('abibas') + return dict(version=distribution.version, + author='Taryel Hlontsi', + license='copyleft (GPLv3)') diff --git a/src/data/controller-models.json b/src/data/controller-models.json index 0d34fdf..08b7f22 100644 --- a/src/data/controller-models.json +++ b/src/data/controller-models.json @@ -1,4 +1,141 @@ [ + { + "Name": "Ads", + "RunAs": "Client", + "Registers": [ + { + "CanOverride": false, + "Name": "Status2", + "Type": "Input", + "Address": 0, + "DisplayAs": "DropdownVariants", + "PossibleValues": [ + { + "NotOperting": 0 + }, + { + "Operating": 1 + } + ] + }, + { + "CanOverride": false, + "Name": "Sim160", + "Type": "Input", + "Address": 928, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "ConfigUnixTime4", + "Type": "Input", + "Address": 259, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "ConfigUnixTime3", + "Type": "Input", + "Address": 258, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "ConfigUnixTime2", + "Type": "Input", + "Address": 257, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "ConfigUnixTime1", + "Type": "Input", + "Address": 256, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "TurbinesConfigured", + "Type": "Input", + "Address": 261, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "InterfaceVersion", + "Type": "Input", + "Address": 260, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "Sim2", + "Type": "Input", + "Address": 770, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "Sim1", + "Type": "Input", + "Address": 769, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": false, + "Name": "SystemStatusCode", + "Type": "Input", + "Address": 1, + "DisplayAs": "RawBinary", + "PossibleValues": [] + }, + { + "CanOverride": true, + "Name": "Interface", + "Type": "Holding", + "Address": 512, + "PossibleValues": [ + { + "Deactivated": 0 + }, + { + "AircraftDetected": 1 + }, + { + "AircraftNotDetected": 3 + } + ] + }, + { + "CanOverride": false, + "Name": "Status", + "Type": "Holding", + "Address": 0, + "PossibleValues": [ + { + "NotOperting": 0 + }, + { + "Operating": 1 + } + ] + } + ], + "DisplayVariants": [ + "DropdownVariants", + "RawBinary", + "RawInteger" + ] + }, { "Name": "CIP300", "RunAs": "Server", @@ -108,43 +245,6 @@ } ] }, - { - "Name": "Ads", - "RunAs": "Client", - "Registers": [ - { - "CanOverride": true, - "Name": "Interface", - "Type": "Holding", - "Address": 512, - "PossibleValues": [ - { - "Deactivated": 0 - }, - { - "AircraftDetected": 1 - }, - { - "AircraftNotDetected": 3 - } - ] - }, - { - "CanOverride": false, - "Name": "Status", - "Type": "Holding", - "Address": 0, - "PossibleValues": [ - { - "NotOperting": 0 - }, - { - "Operating": 1 - } - ] - } - ] - }, { "Name": "CIP400", "RunAs": "Server", diff --git a/src/data/controllers.json b/src/data/controllers.json index fd62e80..34c8463 100644 --- a/src/data/controllers.json +++ b/src/data/controllers.json @@ -11,12 +11,6 @@ "Ip": "127.0.0.1", "Port": 11507 }, - { - "Type": "Quantec", - "Enabled": true, - "Ip": "127.0.0.1", - "Port": 11701 - }, { "Type": "Quantec", "Enabled": true, @@ -28,5 +22,11 @@ "Enabled": true, "Ip": "127.0.0.1", "Port": 11508 + }, + { + "Type": "CIP400", + "Enabled": true, + "Ip": "127.0.0.1", + "Port": 11506 } ] \ No newline at end of file diff --git a/src/main/__init__.py b/src/main/__init__.py index 2e0c00a..36b5058 100644 --- a/src/main/__init__.py +++ b/src/main/__init__.py @@ -11,8 +11,7 @@ def run(): def run_debug(): ui = _run_flask_debug() print(f'UI started: PID {ui.pid}') - #ui.wait() - _run_simulator('info') + _run_simulator('debug') def _run_flask_debug() -> subprocess.Popen: diff --git a/src/modbus/simulator.py b/src/modbus/simulator.py index 75708b8..09f65dd 100755 --- a/src/modbus/simulator.py +++ b/src/modbus/simulator.py @@ -11,6 +11,7 @@ import signal import data import sys import traceback +import plotext as plt _RUN_FOR_SEC = 10 _LOG_LEVEL = 20 # INFO @@ -18,6 +19,8 @@ _LOG_LEVEL = 20 # INFO _statistic = {} _start_time = None _active = {} +_inbound_read_freq = {} +_inbound_write_freq = {} def get_run_options(): @@ -79,22 +82,38 @@ def handle_client(instance, ct: dict): filter(lambda x: x['Type'] == 'Holding' and x['CanOverride'] is True, registers), None) + inputRegs = filter(lambda x: x['Type'] == 'Input', registers) + regs = {} for r in registers: regs[r['Address']] = r['Name'] info = {'Model': ct['Name'], 'Port': instance.port, 'Registers': regs} + if holdingRead: result = instance.read_holding_registers(holdingRead['Address'], 1) - if not result: + if result is None: logging.debug(f'{ct["Name"]} client@{instance.port} no connection') else: gather('read', 'holding', holdingRead['Address'], instance.port, result[0], info, 'outbound') + data.set_controller_register(info['Model'], info['Port'], + holdingRead['Name'], result[0]) if holdingWrite: v = holdingWrite['CurrentValue'] instance.write_single_register(holdingWrite['Address'], v) gather('write', 'holding', holdingWrite['Address'], instance.port, v, info, 'outbound') + for r in list(inputRegs): + result = instance.read_input_registers(r['Address']) + if result is not None: + gather('read', 'input', r['Address'], instance.port, + result[0], info, 'outbound') + data.set_controller_register(info['Model'], info['Port'], + r['Name'], result[0]) + else: + logging.warning( + f'Input register {r["Name"]} on server {instance.port} is misconfigured. Read result is None' + ) else: logging.warning( f'{ct["Name"]} client@{instance.port} is not fully configured') @@ -186,11 +205,13 @@ def gather(op, registry, address, port, value, info, tag='inbound'): Returns ------- - void + None """ global _active global _statistic + global _inbound_read_freq + global _inbound_write_freq controller = f'{info["Model"]}@{info["Port"]}' try: # client port here is a port of incoming connection @@ -216,6 +237,104 @@ def gather(op, registry, address, port, value, info, tag='inbound'): else: logging.debug(msg) + freq_key = f'{op};{registry} {address};{controller}' + if tag == 'inbound' and (op == 'read' or op == 'write'): + collection = _inbound_read_freq if op == 'read' else _inbound_write_freq + entry = collection.get(freq_key) + if entry: + entry[0] += 1 + else: + collection[freq_key] = {0: 0} + + +def calculate_freq(interval_sec: int, dt: datetime.datetime): + global _inbound_read_freq + global _inbound_write_freq + complete_freq_period(_inbound_read_freq, interval_sec, dt) + complete_freq_period(_inbound_write_freq, interval_sec, dt) + + +def complete_freq_period(collection: dict, interval_sec: int, + dt: datetime.datetime): + + format = '%Y-%d-%m %H:%M:%S' + dt_str = dt.strftime(format) + hour_ago_str = (dt - datetime.timedelta(hours=1)).strftime(format) + for k, v in collection.items(): + v[dt_str] = v[0] / interval_sec + v[0] = 0 + keys_to_del = [] + # starting from 3.7 insertion order in dict is preserved + for entry, _ in v.items(): + if entry == 0: + continue + if entry < hour_ago_str: + keys_to_del.append(entry) + else: + break + for entry in keys_to_del: + del v[entry] + + +def merge_dict(base: dict, incoming: dict) -> dict: + result = {**base, **incoming} + for key in result.keys(): + if key in base and key in incoming: + if isinstance(base[key], list) and isinstance(incoming[key], list): + result[key] = base[key] + incoming[key] + elif isinstance(base[key], list): + result[key] = [*base[key], incoming[key]] + elif isinstance(incoming[key], list): + result[key] = [base[key], *incoming[key]] + else: + result[key] = [base[key], incoming[key]] + else: + if not isinstance(result[key], list): + result[key] = [result[key]] + return result + + +def aggregate_freq(collection: dict): + for k, v in collection.items(): + del v[0] + # print(f'{k}:') + # for timestamp, freq in v.items(): + # print(f'{timestamp} {freq} ') + merged = {} + for k, v in collection.items(): + merged = merge_dict(merged, v) + aggregated = {} + for k, v in merged.items(): + v = round(sum(v) / len(v), 2) + aggregated[k] = v + return aggregated + + +def report_freq(): + global _inbound_read_freq + global _inbound_write_freq + + reads = aggregate_freq(_inbound_read_freq) + writes = aggregate_freq(_inbound_write_freq) + if not reads and not writes: + return + + plt.date_form('Y-d-m H:M:S', 'M:S') + if reads: + plt.plot(reads.keys(), + reads.values(), + label='inbound read frequency', + color='blue') + if writes: + plt.plot(writes.keys(), + writes.values(), + label='inbound write frequency', + color='magenta') + plt.title('Inbound Read/Write Frequency') + plt.xlabel('Time (max 1 hour long)') + plt.ylabel('Operations per second') + plt.show() + def report(): global _statistic @@ -300,12 +419,12 @@ class MaBank(DataBank): def on_coils_change(self, address, from_value, to_value, srv_info): gather('change event', 'coils', address, srv_info.client.port, - f'{from_value}⟶{to_value}', self.info) + f'{from_value}→{to_value}', self.info) def on_holding_registers_change(self, address, from_value, to_value, srv_info): gather('change event', 'holding', address, srv_info.client.port, - f'{from_value}⟶{to_value}', self.info) + f'{from_value}→{to_value}', self.info) if __name__ == '__main__': @@ -316,7 +435,7 @@ if __name__ == '__main__': level=_LOG_LEVEL) _start_time = datetime.datetime.now() - version = data.get_version() + version = data.get_info()['version'] logging.info(f'abibas v{version}. Light controllers MODBUS simulator') while run(): @@ -339,12 +458,14 @@ if __name__ == '__main__': else: handle_client(instance, c['Model']) sleep(5) + calculate_freq(5, datetime.datetime.now()) report() + report_freq() dispose() - except Exception: + except Exception as ex: exc = sys.exception() traceback.print_tb(exc.__traceback__, limit=3, file=sys.stdout) - logging.critical('An epic failure just has happened!') + logging.critical(f'An epic failure just has happened! ({ex})') dispose() diff --git a/src/ui/controller_types.py b/src/ui/controller_types.py index 3140215..876a0ce 100644 --- a/src/ui/controller_types.py +++ b/src/ui/controller_types.py @@ -15,7 +15,9 @@ bp = Blueprint('controller_types', __name__) @bp.route('/types') def list(): models = data.get_controller_types() - return render_template('controller_types/list.html', config=models) + return render_template('controller_types/list.html', + config=models, + footer=data.get_info()) @bp.route('/types/add', methods=("GET", "POST")) @@ -34,7 +36,8 @@ def add(): return redirect( url_for('controller_types.edit', name=request.form['name'])) - return render_template('controller_types/create.html') + return render_template('controller_types/create.html', + footer=data.get_info()) @bp.route('/types//edit', methods=("GET", "POST")) @@ -65,7 +68,9 @@ def edit(name: str): existing = _prepare_for_editing(existing['Name']) break - return render_template('controller_types/update.html', data=existing) + return render_template('controller_types/update.html', + data=existing, + footer=data.get_info()) @bp.route('/types//copy', methods=("GET", )) @@ -94,6 +99,7 @@ def _find_model_by_name(name: str, models: dict = None) -> dict: def _prepare_for_editing(name: str): types = ['Input', 'Holding', 'Coil', 'Discrete'] + variants = ['DropdownVariants', 'RawBinary', 'RawInteger'] exists = _find_model_by_name(name) if not exists: return None @@ -104,6 +110,7 @@ def _prepare_for_editing(name: str): processed.append(f'{k}={v}') reg['PossibleValues'] = '\n'.join(processed) exists['RegTypes'] = types + exists['DisplayVariants'] = variants return exists @@ -113,7 +120,10 @@ def _save_model(model: dict, current_name: str): del model['RegTypes'] _remove_reg(model, '') for r in model['Registers']: - r['PossibleValues'] = _str_to_possible(r['PossibleValues']) + if 'PossibleValues' in r: + r['PossibleValues'] = _str_to_possible(r['PossibleValues']) + else: + r['PossibleValues'] = [] models.insert(0, model) data.set_controller_types(models) @@ -146,6 +156,8 @@ def _update_reg(model: dict, reg_name: str, form: dict): updated['CanOverride'] = v == 'True' elif k == f'{reg_name}-type': updated['Type'] = v + elif k == f'{reg_name}-display-as': + updated['DisplayAs'] = v elif k == f'{reg_name}-possible': updated['PossibleValues'] = _str_to_possible(v) _remove_reg(model, reg_name) diff --git a/src/ui/controllers.py b/src/ui/controllers.py index 51a0510..d139266 100644 --- a/src/ui/controllers.py +++ b/src/ui/controllers.py @@ -1,5 +1,6 @@ """Setting up light controllers and monitoring their state""" +import re import data from flask import (Blueprint, flash, g, redirect, render_template, request, url_for) @@ -10,8 +11,10 @@ bp = Blueprint('controllers', __name__) @bp.route('/', methods=('GET', )) def list(): - data = get_list() - return render_template('controllers/index.html', config=data) + lst = get_list() + return render_template('controllers/index.html', + config=lst, + footer=data.get_info()) @bp.route('/override', methods=('POST', )) @@ -27,13 +30,22 @@ def update(): if k == 'enabled': enabled = v continue - registers.append({'Name': k, 'Value': int(v)}) + registers.append({'Name': k, 'Value': v}) controllers = data.get_controllers() existing = _find(port, controllers) if existing is None: abort(404) + current_regs = data.get_controller_registers(existing['Port']) + for r in registers: + reg = next(filter(lambda x: x['Name'] == r['Name'], current_regs), r) + display_as = reg.get('DisplayAs', 'DropdownVariants') + if display_as == 'RawBinary' and _is_binary_str(r['Value']): + r['Value'] = int(r['Value'], 2) + else: + r['Value'] = int(r['Value']) + data.set_controller_registers(existing['Type'], existing['Port'], registers) enabled = enabled == 'True' @@ -89,15 +101,31 @@ def add(): data.set_controllers(controllers) return redirect(url_for('controllers.list')) - return render_template('controllers/create.html', types=types) + return render_template('controllers/create.html', + types=types, + footer=data.get_info()) -def _get_value_name(value: int, possibleValues: list) -> str: - for possible in possibleValues: - for k, v in possible.items(): - if v == value: - return k - return 'Unknown' +def _get_current_value(reg: dict) -> str: + if 'DisplayAs' in reg and reg['DisplayAs'] == 'RawBinary': + return f'{reg["CurrentValue"]:b}'.zfill(16) + else: + return reg['CurrentValue'] + + +def _get_value_name(reg: dict) -> str: + value = reg['CurrentValue'] + display_as = reg.get('DisplayAs', 'DropdownVariants') + if display_as == 'RawBinary': + return value + elif display_as == 'RawInteger': + return '' + else: + for possible in reg['PossibleValues']: + for k, v in possible.items(): + if v == value: + return k + return 'Undefined' def get_list() -> dict: @@ -113,9 +141,9 @@ def get_list() -> dict: 'Name': reg['Name'], 'Value': - reg['CurrentValue'], + _get_current_value(reg), 'ValueStr': - _get_value_name(reg['CurrentValue'], reg['PossibleValues']), + _get_value_name(reg), 'Address': reg['Address'], 'Type': @@ -123,7 +151,9 @@ def get_list() -> dict: 'PossibleValues': reg['PossibleValues'], 'CanOverride': - reg['CanOverride'] + reg['CanOverride'], + 'DisplayAs': + reg.get('DisplayAs', 'DropdownVariants') }) return controllers @@ -134,3 +164,8 @@ def _to_status(enabled: bool) -> str: return 'Yes' else: return 'No' + + +def _is_binary_str(s: str) -> bool: + pattern = r'^[01]+$' + return re.match(pattern, s) is not None diff --git a/src/ui/static/style.css b/src/ui/static/style.css index be3b820..766edff 100644 --- a/src/ui/static/style.css +++ b/src/ui/static/style.css @@ -1,8 +1,12 @@ html { font-family: sans-serif; background: #eee; - padding: 1rem; - margin: 0.5rem; +} + +body { + min-height: 100vh; + display: flex; + flex-direction: column; } nav { @@ -186,3 +190,25 @@ label { flex-basis: 100%; /* Take full width of the container */ } } + +footer { + margin-top: auto; + height: 2rem; + text-align: center; + background-color: #e5bdf1; + padding: 0.5rem 0.5rem; + opacity: 0.50; +} + +footer p { + margin: 0; +} + +.grey-box { + background-color: #d0d0d7; + padding: 0.2rem; + border: 1px solid #ccc; + display: inline-block; + font-size: 0.7rem; + border-radius: 0.3rem +} diff --git a/src/ui/templates/base.html b/src/ui/templates/base.html index 75a0577..3c0eed4 100644 --- a/src/ui/templates/base.html +++ b/src/ui/templates/base.html @@ -5,6 +5,7 @@ {% block title %}{% endblock %} +