Commit 640c0130 authored by Kirill Smelkov's avatar Kirill Smelkov

slapos/promise/plugin/check_cpri_lock: Don't check whatever device blindly

Currently this promise is implemented by grepping whole rf_info output
for "HW" and "SW" strings. But this won't work ok in the presence of
multiple CPRI devices. Imagine, for example if one device has CPRI lock,
while the other does not:

    PCIe CPRI /dev/sdr2@1:
      Hardware ID: 0x4b12
      DNA: [0x0048248a334a7054]
      Serial: ''
      FPGA revision: 2023-06-23  10:05:24
      FPGA vccint: 0.98 V
      FPGA vccaux: 1.76 V
      FPGA vccbram: 0.98 V
      FPGA temperature: 71.9 °C
      Clock tune: 0.0 ppm
      NUMA: 0
      CPRI_option: '5' (x8) lock=no                     <-- NOTE
      DMA0: TX fifo: 66.67us  Usage=16/32768 (0%)
      DMA0: RX fifo: 66.67us  Usage=16/32768 (0%)
      DMA0 Underflows: 0
      DMA0 Overflows: 0
    PCIe CPRI /dev/sdr3@1:
      Hardware ID: 0x4b12
      DNA: [0x0048248a334a7054]
      Serial: ''
      FPGA revision: 2023-06-23  10:05:24
      FPGA vccint: 0.98 V
      FPGA vccaux: 1.77 V
      FPGA vccbram: 0.98 V
      FPGA temperature: 71.7 °C
      Clock tune: 0.0 ppm
      NUMA: 0
      CPRI_option: '5' (x8) lock=HW+SW rx/tx=46.606us   <-- NOTE
        Port #0: T14=46.606us
      DMA0: TX fifo: 66.67us  Usage=16/32768 (0%)
      DMA0: RX fifo: 66.67us  Usage=16/32768 (0%)
      DMA0 Underflows: 0
      DMA0 Overflows: 0

the old code would still report "CPRI locked all ok" and also globally
without indicating which CPRI channel is locked.

-> Fix it by adjusting check_cpri_lock to parse rf_info text more
precisely, detect devices there and to understand which device has CPRI
lock and which does not.

For now this change is accompanied by the following change in
ors-amarisoft SR to keep it working:

    --- a/software/ors-amarisoft/instance-enb.jinja2.cfg
    +++ b/software/ors-amarisoft/instance-enb.jinja2.cfg
    @@ -35,7 +35,6 @@ parts =
       check-lopcomm-sync.py
       check-lopcomm-config-log.py
       check-lopcomm-stats-log.py
    -  check-cpri-lock.py
     {% endif %}
     {% if slapparameter_dict.get("dnsmasq", None) %}
       dnsmasq-service
    @@ -48,6 +47,7 @@ parts =
     {% endif %}
       monitor-base
       publish-connection-information
    +{% set extra_part_list = [] %}

     extends = {{ monitor_template }}

    @@ -688,12 +688,21 @@ config-testing = {{ slapparameter_dict.get("testing", False) }}
     config-config-log = ${lopcomm-rrh-config-template:log-output}
     config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}

    -[check-cpri-lock.py]
    +{%  if ru == "lopcomm" %}
    +{%-   set cell_list = slapparameter_dict.get('cell_list', {'default': {}}) %}
    +{%-   for i, k in enumerate(cell_list) %}
    +{%-     set sfp_port = cell_list[k].get('cpri_port_number', i) %}
    +{%-     do extra_part_list.append('SFP{{sfp_port}}-cpri-lock.py') %}
    +[SFP{{sfp_port}}-cpri-lock.py]
     <= macro.promise
     promise = check_cpri_lock
     config-testing = {{ slapparameter_dict.get("testing", False) }}
    +config-sdr_dev  = {{ slapparameter_dict.get('sdr_number', 0) }}
    +config-sfp_port = {{ sfp_port }}
     config-amarisoft-rf-info-log = ${amarisoft-rf-info-template:log-output}
     config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
    +{%-  endfor %}
    +{% endif %}

     [check-rx-saturated.py]
     <= macro.promise
    @@ -702,3 +711,9 @@ config-testing = {{ slapparameter_dict.get("testing", False) }}
     config-amarisoft-stats-log = ${amarisoft-stats-template:log-output}
     config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
     config-max-rx-sample-db = {{ slapparameter_dict.get("max_rx_sample_db", 0) }}
    +
    +[buildout]
    +parts +=
    +{%- for part in extra_part_list %}
    +    {{ part }}
    +{%- endfor %}

(posted in slapos!1461)

The way rf_info text is parsed could be also useful in the future to
e.g. detect FPGA revision of the boards and report their recency status
via promise.

/cc @jhuge, @tomo, @xavier_thompson, @Daetalus
/reviewed-by @lu.xu
/reviewed-on !127
parent 7c3b240f
import errno
import json
import logging
import os
from dateutil import parser
import re
from .util import JSONPromise, get_json_log_data_interval
from zope.interface import implementer
......@@ -15,6 +10,7 @@ class RunPromise(JSONPromise):
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=1)
self.amarisoft_rf_info_log = self.getConfig('amarisoft-rf-info-log')
self.sdr_devchan = "/dev/sdr%s@%s" % (self.getConfig('sdr_dev'), self.getConfig('sfp_port'))
self.stats_period = int(self.getConfig('stats-period'))
self.testing = self.getConfig('testing') == "True"
......@@ -22,24 +18,78 @@ class RunPromise(JSONPromise):
if self.testing:
self.logger.info("skipping promise")
return
def error(msg): self.logger.error("%s: %s", self.sdr_devchan, msg)
def info(msg): self.logger.info ("%s: %s", self.sdr_devchan, msg)
data_list = get_json_log_data_interval(self.amarisoft_rf_info_log, self.stats_period * 2)
if len(data_list) < 1:
self.logger.error("rf_info: stale data")
error("rf_info: stale data")
return
rf_info_text = data_list[0]['rf_info']
if "CPRI" not in rf_info_text:
self.logger.info("No CPRI feature")
else:
if "HW" in rf_info_text and "SW" in rf_info_text:
self.logger.info("CPRI locked")
else:
if "HW" not in rf_info_text:
self.logger.error("HW Lock is missing")
if "SW" not in rf_info_text:
self.logger.error("SW Lock is missing")
rf_info = self._parse_rf_info(rf_info_text)
if self.sdr_devchan not in rf_info:
error("rf_info: no device entry")
return
rf_info = rf_info[self.sdr_devchan]
icpri = rf_info.get('CPRI_option')
if icpri is None:
error("no CPRI feature")
return
hw = ("HW" in icpri)
sw = ("SW" in icpri)
if not hw:
error("HW Lock is missing")
if not sw:
error("SW Lock is missing")
if hw and sw:
info("CPRI locked")
@staticmethod
def _parse_rf_info(rf_info_text): # -> {} /dev/sdrX@Y -> {key: value}
"""_parse_rf_info parses rf_info output into per-SDR-device key->value dictionaries.
For example:
TRX SDR driver 2023-09-07, API v15/18
PCIe CPRI /dev/sdr1@2:
FPGA vccint: 0.98 V
FPGA vccaux: 1.77 V
PCIe CPRI /dev/sdr3@4:
ABC: 123
DEF: 4567
is parsed as {'/dev/sdr1@2': {'FPGA vccint': '0.98 V', 'FPGA vccaux': '1.77 V'},
'/dev/sdr3@4': {'ABC': '123', 'DEF': '4567'}}
"""
rf_info = {}
cur = None
for l in rf_info_text.splitlines():
if not l.startswith(' '): # possibly start of new /dev entry
cur = None
m = re.search(r' (/dev/sdr[^\s]+):$', l)
if m is None: # not so - ignore the line
continue
cur = {}
sdr_devchan = m.group(1)
rf_info[sdr_devchan] = cur
continue
# indented line - it populates current if it still holds its context
if cur is None:
continue
k, v = l.split(':', 1)
k = k.strip()
v = v.strip()
cur[k] = v
return rf_info
def test(self):
"""
Called after sense() if the instance is still converging.
......
......@@ -44,6 +44,47 @@ class TestCheckCpriLock(TestPromisePluginMixin):
super(TestCheckCpriLock, self).setUp()
self.amarisoft_rf_info_log = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'amarisoft_rf_info.json.log')
rf_info = \
"""
TRX SDR driver 2023-09-07, API v15/18
PCIe CPRI /dev/sdr2@1:
Hardware ID: 0x4b12
DNA: [0x0048248a334a7054]
Serial: ''
FPGA revision: 2023-06-23 10:05:24
FPGA vccint: 0.98 V
FPGA vccaux: 1.76 V
FPGA vccbram: 0.98 V
FPGA temperature: 71.9 °C
Clock tune: 0.0 ppm
NUMA: 0
CPRI_option: '5' (x8) lock=no
DMA0: TX fifo: 66.67us Usage=16/32768 (0%)
DMA0: RX fifo: 66.67us Usage=16/32768 (0%)
DMA0 Underflows: 0
DMA0 Overflows: 0
PCIe CPRI /dev/sdr3@1:
Hardware ID: 0x4b12
DNA: [0x0048248a334a7054]
Serial: ''
FPGA revision: 2023-06-23 10:05:24
FPGA vccint: 0.98 V
FPGA vccaux: 1.77 V
FPGA vccbram: 0.98 V
FPGA temperature: 71.7 °C
Clock tune: 0.0 ppm
NUMA: 0
CPRI_option: '5' (x8) lock=HW+SW rx/tx=46.606us
Port #0: T14=46.606us
DMA0: TX fifo: 66.67us Usage=16/32768 (0%)
DMA0: RX fifo: 66.67us Usage=16/32768 (0%)
DMA0 Underflows: 0
DMA0 Overflows: 0
PCIe SDR /dev/sdr4@0:
AAA: bbb
"""
self.rf_info_data = {'message': 'rf', 'rf_info': rf_info}
def writeLog(self, data, ago=5):
with open(self.amarisoft_rf_info_log, 'w') as f:
......@@ -59,21 +100,35 @@ class TestCheckCpriLock(TestPromisePluginMixin):
% (RunPromise.__module__, RunPromise.__name__, kw))
def test_locked_ok(self):
self.writeLog({'rf_info': "CPRI: x16 HW SW"})
self.writePromise()
self.writeLog(self.rf_info_data)
self.writePromise(sdr_dev='3', sfp_port='1')
self.configureLauncher()
self.launcher.run()
def test_no_lock(self):
self.writeLog({'rf_info': "CPRI: x16\\n"})
self.writePromise()
self.writeLog(self.rf_info_data)
self.writePromise(sdr_dev='2', sfp_port='1')
self.configureLauncher()
with self.assertRaisesRegex(PromiseError, r'(?m)HW Lock is missing\n.*SW Lock is missing'):
self.launcher.run()
def test_no_device(self):
self.writeLog(self.rf_info_data)
self.writePromise(sdr_dev='1', sfp_port='0')
self.configureLauncher()
with self.assertRaisesRegex(PromiseError, 'no device entry'):
self.launcher.run()
def test_no_cpri_entry(self):
self.writeLog(self.rf_info_data)
self.writePromise(sdr_dev='4', sfp_port='0')
self.configureLauncher()
with self.assertRaises(PromiseError):
with self.assertRaisesRegex(PromiseError, 'no CPRI feature'):
self.launcher.run()
def test_stale_data(self):
self.writeLog({'rf_info': "CPRI: x16 HW SW"}, ago=500)
self.writePromise()
self.writeLog(self.rf_info_data, ago=500)
self.writePromise(sdr_dev='3', sfp_port='1')
self.configureLauncher()
with self.assertRaisesRegex(PromiseError, 'rf_info: stale data'):
self.launcher.run()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment