Commit 2e41c380 authored by David S. Miller's avatar David S. Miller

Merge branch 'devlink-Add-configuration-parameters-support'

Moshe Shemesh says:

====================
Add configuration parameters support

Add configuration parameters setting through devlink.
Each device registers supported configuration parameters table.
Each parameter can be either generic or driver specific.
The user can retrieve data on these parameters by "devlink param show"
command and can set new value to a parameter by "devlink param set"
command.
The parameters can be set in different configuration modes:
  runtime - set while driver is running, no reset required.
  driverinit - applied while driver initializes, requires restart
               driver by devlink reload command.
  permanent - written to device's non-volatile memory, hard reset required.

The patches at the end of the patchset introduce few params that are using
the introduced infrastructure on mlx4 and bnxt.

Command examples and output:

pci/0000:81:00.0:
  name internal_error_reset type generic
    values:
      cmode runtime value true
      cmode driverinit value true
  name max_macs type generic
    values:
      cmode driverinit value 128
  name enable_64b_cqe_eqe type driver-specific
    values:
      cmode driverinit value true
  name enable_4k_uar type driver-specific
    values:
      cmode driverinit value false

pci/0000:81:00.0:
  name internal_error_reset type generic
    values:
      cmode runtime value false
      cmode driverinit value true
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 827ad90c 6354b95e
......@@ -21,16 +21,99 @@ static const struct devlink_ops bnxt_dl_ops = {
#endif /* CONFIG_BNXT_SRIOV */
};
static const struct bnxt_dl_nvm_param nvm_params[] = {
{DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV, NVM_OFF_ENABLE_SRIOV,
BNXT_NVM_SHARED_CFG, 1},
};
static int bnxt_hwrm_nvm_req(struct bnxt *bp, u32 param_id, void *msg,
int msg_len, union devlink_param_value *val)
{
struct hwrm_nvm_variable_input *req = msg;
void *data_addr = NULL, *buf = NULL;
struct bnxt_dl_nvm_param nvm_param;
int bytesize, idx = 0, rc, i;
dma_addr_t data_dma_addr;
/* Get/Set NVM CFG parameter is supported only on PFs */
if (BNXT_VF(bp))
return -EPERM;
for (i = 0; i < ARRAY_SIZE(nvm_params); i++) {
if (nvm_params[i].id == param_id) {
nvm_param = nvm_params[i];
break;
}
}
if (nvm_param.dir_type == BNXT_NVM_PORT_CFG)
idx = bp->pf.port_id;
else if (nvm_param.dir_type == BNXT_NVM_FUNC_CFG)
idx = bp->pf.fw_fid - BNXT_FIRST_PF_FID;
bytesize = roundup(nvm_param.num_bits, BITS_PER_BYTE) / BITS_PER_BYTE;
if (nvm_param.num_bits == 1)
buf = &val->vbool;
data_addr = dma_zalloc_coherent(&bp->pdev->dev, bytesize,
&data_dma_addr, GFP_KERNEL);
if (!data_addr)
return -ENOMEM;
req->data_addr = cpu_to_le64(data_dma_addr);
req->data_len = cpu_to_le16(nvm_param.num_bits);
req->option_num = cpu_to_le16(nvm_param.offset);
req->index_0 = cpu_to_le16(idx);
if (idx)
req->dimensions = cpu_to_le16(1);
if (req->req_type == HWRM_NVM_SET_VARIABLE)
memcpy(data_addr, buf, bytesize);
rc = hwrm_send_message(bp, msg, msg_len, HWRM_CMD_TIMEOUT);
if (!rc && req->req_type == HWRM_NVM_GET_VARIABLE)
memcpy(buf, data_addr, bytesize);
dma_free_coherent(&bp->pdev->dev, bytesize, data_addr, data_dma_addr);
if (rc)
return -EIO;
return 0;
}
static int bnxt_dl_nvm_param_get(struct devlink *dl, u32 id,
struct devlink_param_gset_ctx *ctx)
{
struct hwrm_nvm_get_variable_input req = {0};
struct bnxt *bp = bnxt_get_bp_from_dl(dl);
bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_NVM_GET_VARIABLE, -1, -1);
return bnxt_hwrm_nvm_req(bp, id, &req, sizeof(req), &ctx->val);
}
static int bnxt_dl_nvm_param_set(struct devlink *dl, u32 id,
struct devlink_param_gset_ctx *ctx)
{
struct hwrm_nvm_set_variable_input req = {0};
struct bnxt *bp = bnxt_get_bp_from_dl(dl);
bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_NVM_SET_VARIABLE, -1, -1);
return bnxt_hwrm_nvm_req(bp, id, &req, sizeof(req), &ctx->val);
}
static const struct devlink_param bnxt_dl_params[] = {
DEVLINK_PARAM_GENERIC(ENABLE_SRIOV,
BIT(DEVLINK_PARAM_CMODE_PERMANENT),
bnxt_dl_nvm_param_get, bnxt_dl_nvm_param_set,
NULL),
};
int bnxt_dl_register(struct bnxt *bp)
{
struct devlink *dl;
int rc;
if (!pci_find_ext_capability(bp->pdev, PCI_EXT_CAP_ID_SRIOV))
return 0;
if (bp->hwrm_spec_code < 0x10803) {
netdev_warn(bp->dev, "Firmware does not support SR-IOV E-Switch SWITCHDEV mode.\n");
if (bp->hwrm_spec_code < 0x10600) {
netdev_warn(bp->dev, "Firmware does not support NVM params");
return -ENOTSUPP;
}
......@@ -41,16 +124,34 @@ int bnxt_dl_register(struct bnxt *bp)
}
bnxt_link_bp_to_dl(bp, dl);
/* Add switchdev eswitch mode setting, if SRIOV supported */
if (pci_find_ext_capability(bp->pdev, PCI_EXT_CAP_ID_SRIOV) &&
bp->hwrm_spec_code > 0x10803)
bp->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
rc = devlink_register(dl, &bp->pdev->dev);
if (rc) {
bnxt_link_bp_to_dl(bp, NULL);
devlink_free(dl);
netdev_warn(bp->dev, "devlink_register failed. rc=%d", rc);
return rc;
goto err_dl_free;
}
rc = devlink_params_register(dl, bnxt_dl_params,
ARRAY_SIZE(bnxt_dl_params));
if (rc) {
netdev_warn(bp->dev, "devlink_params_register failed. rc=%d",
rc);
goto err_dl_unreg;
}
return 0;
err_dl_unreg:
devlink_unregister(dl);
err_dl_free:
bnxt_link_bp_to_dl(bp, NULL);
devlink_free(dl);
return rc;
}
void bnxt_dl_unregister(struct bnxt *bp)
......@@ -60,6 +161,8 @@ void bnxt_dl_unregister(struct bnxt *bp)
if (!dl)
return;
devlink_params_unregister(dl, bnxt_dl_params,
ARRAY_SIZE(bnxt_dl_params));
devlink_unregister(dl);
devlink_free(dl);
}
......@@ -33,6 +33,21 @@ static inline void bnxt_link_bp_to_dl(struct bnxt *bp, struct devlink *dl)
}
}
#define NVM_OFF_ENABLE_SRIOV 401
enum bnxt_nvm_dir_type {
BNXT_NVM_SHARED_CFG = 40,
BNXT_NVM_PORT_CFG,
BNXT_NVM_FUNC_CFG,
};
struct bnxt_dl_nvm_param {
u16 id;
u16 offset;
u16 dir_type;
u16 num_bits;
};
int bnxt_dl_register(struct bnxt *bp);
void bnxt_dl_unregister(struct bnxt *bp);
......
......@@ -6201,6 +6201,19 @@ struct hwrm_nvm_install_update_cmd_err {
u8 unused_0[7];
};
struct hwrm_nvm_variable_input {
__le16 req_type;
__le16 cmpl_ring;
__le16 seq_id;
__le16 target_id;
__le64 resp_addr;
__le64 data_addr;
__le16 data_len;
__le16 option_num;
__le16 dimensions;
__le16 index_0;
};
/* hwrm_nvm_get_variable_input (size:320b/40B) */
struct hwrm_nvm_get_variable_input {
__le16 req_type;
......
......@@ -543,9 +543,14 @@ int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode)
break;
case DEVLINK_ESWITCH_MODE_SWITCHDEV:
if (bp->hwrm_spec_code < 0x10803) {
netdev_warn(bp->dev, "FW does not support SRIOV E-Switch SWITCHDEV mode\n");
rc = -ENOTSUPP;
goto done;
}
if (pci_num_vf(bp->pdev) == 0) {
netdev_info(bp->dev,
"Enable VFs before setting switchdev mode");
netdev_info(bp->dev, "Enable VFs before setting switchdev mode");
rc = -EPERM;
goto done;
}
......
......@@ -212,7 +212,7 @@ static void mlx4_handle_error_state(struct mlx4_dev_persistent *persist)
mutex_lock(&persist->interface_state_mutex);
if (persist->interface_state & MLX4_INTERFACE_STATE_UP &&
!(persist->interface_state & MLX4_INTERFACE_STATE_DELETION)) {
err = mlx4_restart_one(persist->pdev);
err = mlx4_restart_one(persist->pdev, false, NULL);
mlx4_info(persist->dev, "mlx4_restart_one was ended, ret=%d\n",
err);
}
......
......@@ -177,6 +177,101 @@ struct mlx4_port_config {
static atomic_t pf_loading = ATOMIC_INIT(0);
static int mlx4_devlink_ierr_reset_get(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx)
{
ctx->val.vbool = !!mlx4_internal_err_reset;
return 0;
}
static int mlx4_devlink_ierr_reset_set(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx)
{
mlx4_internal_err_reset = ctx->val.vbool;
return 0;
}
static int
mlx4_devlink_max_macs_validate(struct devlink *devlink, u32 id,
union devlink_param_value val,
struct netlink_ext_ack *extack)
{
u32 value = val.vu32;
if (value < 1 || value > 128)
return -ERANGE;
if (!is_power_of_2(value)) {
NL_SET_ERR_MSG_MOD(extack, "max_macs supported must be power of 2");
return -EINVAL;
}
return 0;
}
enum mlx4_devlink_param_id {
MLX4_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
};
static const struct devlink_param mlx4_devlink_params[] = {
DEVLINK_PARAM_GENERIC(INT_ERR_RESET,
BIT(DEVLINK_PARAM_CMODE_RUNTIME) |
BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
mlx4_devlink_ierr_reset_get,
mlx4_devlink_ierr_reset_set, NULL),
DEVLINK_PARAM_GENERIC(MAX_MACS,
BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
NULL, NULL, mlx4_devlink_max_macs_validate),
DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
"enable_64b_cqe_eqe", DEVLINK_PARAM_TYPE_BOOL,
BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
NULL, NULL, NULL),
DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
"enable_4k_uar", DEVLINK_PARAM_TYPE_BOOL,
BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
NULL, NULL, NULL),
};
static void mlx4_devlink_set_init_value(struct devlink *devlink, u32 param_id,
union devlink_param_value init_val)
{
struct mlx4_priv *priv = devlink_priv(devlink);
struct mlx4_dev *dev = &priv->dev;
int err;
err = devlink_param_driverinit_value_set(devlink, param_id, init_val);
if (err)
mlx4_warn(dev,
"devlink set parameter %u value failed (err = %d)",
param_id, err);
}
static void mlx4_devlink_set_params_init_values(struct devlink *devlink)
{
union devlink_param_value value;
value.vbool = !!mlx4_internal_err_reset;
mlx4_devlink_set_init_value(devlink,
DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
value);
value.vu32 = 1UL << log_num_mac;
mlx4_devlink_set_init_value(devlink,
DEVLINK_PARAM_GENERIC_ID_MAX_MACS, value);
value.vbool = enable_64b_cqe_eqe;
mlx4_devlink_set_init_value(devlink,
MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
value);
value.vbool = enable_4k_uar;
mlx4_devlink_set_init_value(devlink,
MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
value);
}
static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev,
struct mlx4_dev_cap *dev_cap)
{
......@@ -3757,8 +3852,57 @@ static int mlx4_devlink_port_type_set(struct devlink_port *devlink_port,
return __set_port_type(info, mlx4_port_type);
}
static void mlx4_devlink_param_load_driverinit_values(struct devlink *devlink)
{
union devlink_param_value saved_value;
int err;
err = devlink_param_driverinit_value_get(devlink,
DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
&saved_value);
if (!err && mlx4_internal_err_reset != saved_value.vbool) {
mlx4_internal_err_reset = saved_value.vbool;
/* Notify on value changed on runtime configuration mode */
devlink_param_value_changed(devlink,
DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET);
}
err = devlink_param_driverinit_value_get(devlink,
DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
&saved_value);
if (!err)
log_num_mac = order_base_2(saved_value.vu32);
err = devlink_param_driverinit_value_get(devlink,
MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
&saved_value);
if (!err)
enable_64b_cqe_eqe = saved_value.vbool;
err = devlink_param_driverinit_value_get(devlink,
MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
&saved_value);
if (!err)
enable_4k_uar = saved_value.vbool;
}
static int mlx4_devlink_reload(struct devlink *devlink,
struct netlink_ext_ack *extack)
{
struct mlx4_priv *priv = devlink_priv(devlink);
struct mlx4_dev *dev = &priv->dev;
struct mlx4_dev_persistent *persist = dev->persist;
int err;
if (persist->num_vfs)
mlx4_warn(persist->dev, "Reload performed on PF, will cause reset on operating Virtual Functions\n");
err = mlx4_restart_one(persist->pdev, true, devlink);
if (err)
mlx4_err(persist->dev, "mlx4_restart_one failed, ret=%d\n", err);
return err;
}
static const struct devlink_ops mlx4_devlink_ops = {
.port_type_set = mlx4_devlink_port_type_set,
.reload = mlx4_devlink_reload,
};
static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
......@@ -3792,14 +3936,21 @@ static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
ret = devlink_register(devlink, &pdev->dev);
if (ret)
goto err_persist_free;
ret = __mlx4_init_one(pdev, id->driver_data, priv);
ret = devlink_params_register(devlink, mlx4_devlink_params,
ARRAY_SIZE(mlx4_devlink_params));
if (ret)
goto err_devlink_unregister;
mlx4_devlink_set_params_init_values(devlink);
ret = __mlx4_init_one(pdev, id->driver_data, priv);
if (ret)
goto err_params_unregister;
pci_save_state(pdev);
return 0;
err_params_unregister:
devlink_params_unregister(devlink, mlx4_devlink_params,
ARRAY_SIZE(mlx4_devlink_params));
err_devlink_unregister:
devlink_unregister(devlink);
err_persist_free:
......@@ -3936,6 +4087,8 @@ static void mlx4_remove_one(struct pci_dev *pdev)
pci_release_regions(pdev);
mlx4_pci_disable_device(dev);
devlink_params_unregister(devlink, mlx4_devlink_params,
ARRAY_SIZE(mlx4_devlink_params));
devlink_unregister(devlink);
kfree(dev->persist);
devlink_free(devlink);
......@@ -3960,7 +4113,7 @@ static int restore_current_port_types(struct mlx4_dev *dev,
return err;
}
int mlx4_restart_one(struct pci_dev *pdev)
int mlx4_restart_one(struct pci_dev *pdev, bool reload, struct devlink *devlink)
{
struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
struct mlx4_dev *dev = persist->dev;
......@@ -3973,6 +4126,8 @@ int mlx4_restart_one(struct pci_dev *pdev)
memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
mlx4_unload_one(pdev);
if (reload)
mlx4_devlink_param_load_driverinit_values(devlink);
err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 1);
if (err) {
mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n",
......
......@@ -1042,7 +1042,8 @@ void mlx4_start_catas_poll(struct mlx4_dev *dev);
void mlx4_stop_catas_poll(struct mlx4_dev *dev);
int mlx4_catas_init(struct mlx4_dev *dev);
void mlx4_catas_end(struct mlx4_dev *dev);
int mlx4_restart_one(struct pci_dev *pdev);
int mlx4_restart_one(struct pci_dev *pdev, bool reload,
struct devlink *devlink);
int mlx4_register_device(struct mlx4_dev *dev);
void mlx4_unregister_device(struct mlx4_dev *dev);
void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type,
......
......@@ -27,6 +27,7 @@ struct devlink {
struct list_head sb_list;
struct list_head dpipe_table_list;
struct list_head resource_list;
struct list_head param_list;
struct devlink_dpipe_headers *dpipe_headers;
const struct devlink_ops *ops;
struct device *dev;
......@@ -295,6 +296,107 @@ struct devlink_resource {
#define DEVLINK_RESOURCE_ID_PARENT_TOP 0
#define DEVLINK_PARAM_MAX_STRING_VALUE 32
enum devlink_param_type {
DEVLINK_PARAM_TYPE_U8,
DEVLINK_PARAM_TYPE_U16,
DEVLINK_PARAM_TYPE_U32,
DEVLINK_PARAM_TYPE_STRING,
DEVLINK_PARAM_TYPE_BOOL,
};
union devlink_param_value {
u8 vu8;
u16 vu16;
u32 vu32;
const char *vstr;
bool vbool;
};
struct devlink_param_gset_ctx {
union devlink_param_value val;
enum devlink_param_cmode cmode;
};
/**
* struct devlink_param - devlink configuration parameter data
* @name: name of the parameter
* @generic: indicates if the parameter is generic or driver specific
* @type: parameter type
* @supported_cmodes: bitmap of supported configuration modes
* @get: get parameter value, used for runtime and permanent
* configuration modes
* @set: set parameter value, used for runtime and permanent
* configuration modes
* @validate: validate input value is applicable (within value range, etc.)
*
* This struct should be used by the driver to fill the data for
* a parameter it registers.
*/
struct devlink_param {
u32 id;
const char *name;
bool generic;
enum devlink_param_type type;
unsigned long supported_cmodes;
int (*get)(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx);
int (*set)(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx);
int (*validate)(struct devlink *devlink, u32 id,
union devlink_param_value val,
struct netlink_ext_ack *extack);
};
struct devlink_param_item {
struct list_head list;
const struct devlink_param *param;
union devlink_param_value driverinit_value;
bool driverinit_value_valid;
};
enum devlink_param_generic_id {
DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
/* add new param generic ids above here*/
__DEVLINK_PARAM_GENERIC_ID_MAX,
DEVLINK_PARAM_GENERIC_ID_MAX = __DEVLINK_PARAM_GENERIC_ID_MAX - 1,
};
#define DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME "internal_error_reset"
#define DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE DEVLINK_PARAM_TYPE_BOOL
#define DEVLINK_PARAM_GENERIC_MAX_MACS_NAME "max_macs"
#define DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE DEVLINK_PARAM_TYPE_U32
#define DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME "enable_sriov"
#define DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE DEVLINK_PARAM_TYPE_BOOL
#define DEVLINK_PARAM_GENERIC(_id, _cmodes, _get, _set, _validate) \
{ \
.id = DEVLINK_PARAM_GENERIC_ID_##_id, \
.name = DEVLINK_PARAM_GENERIC_##_id##_NAME, \
.type = DEVLINK_PARAM_GENERIC_##_id##_TYPE, \
.generic = true, \
.supported_cmodes = _cmodes, \
.get = _get, \
.set = _set, \
.validate = _validate, \
}
#define DEVLINK_PARAM_DRIVER(_id, _name, _type, _cmodes, _get, _set, _validate) \
{ \
.id = _id, \
.name = _name, \
.type = _type, \
.supported_cmodes = _cmodes, \
.get = _get, \
.set = _set, \
.validate = _validate, \
}
struct devlink_ops {
int (*reload)(struct devlink *devlink, struct netlink_ext_ack *extack);
int (*port_type_set)(struct devlink_port *devlink_port,
......@@ -430,6 +532,17 @@ void devlink_resource_occ_get_register(struct devlink *devlink,
void *occ_get_priv);
void devlink_resource_occ_get_unregister(struct devlink *devlink,
u64 resource_id);
int devlink_params_register(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count);
void devlink_params_unregister(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count);
int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
union devlink_param_value *init_val);
int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
union devlink_param_value init_val);
void devlink_param_value_changed(struct devlink *devlink, u32 param_id);
#else
......@@ -622,6 +735,42 @@ devlink_resource_occ_get_unregister(struct devlink *devlink,
{
}
static inline int
devlink_params_register(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count)
{
return 0;
}
static inline void
devlink_params_unregister(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count)
{
}
static inline int
devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
union devlink_param_value *init_val)
{
return -EOPNOTSUPP;
}
static inline int
devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
union devlink_param_value init_val)
{
return -EOPNOTSUPP;
}
static inline void
devlink_param_value_changed(struct devlink *devlink, u32 param_id)
{
return -EOPNOTSUPP;
}
#endif
#endif /* _NET_DEVLINK_H_ */
......@@ -78,6 +78,11 @@ enum devlink_command {
*/
DEVLINK_CMD_RELOAD,
DEVLINK_CMD_PARAM_GET, /* can dump */
DEVLINK_CMD_PARAM_SET,
DEVLINK_CMD_PARAM_NEW,
DEVLINK_CMD_PARAM_DEL,
/* add new commands above here */
__DEVLINK_CMD_MAX,
DEVLINK_CMD_MAX = __DEVLINK_CMD_MAX - 1
......@@ -142,6 +147,16 @@ enum devlink_port_flavour {
*/
};
enum devlink_param_cmode {
DEVLINK_PARAM_CMODE_RUNTIME,
DEVLINK_PARAM_CMODE_DRIVERINIT,
DEVLINK_PARAM_CMODE_PERMANENT,
/* Add new configuration modes above */
__DEVLINK_PARAM_CMODE_MAX,
DEVLINK_PARAM_CMODE_MAX = __DEVLINK_PARAM_CMODE_MAX - 1
};
enum devlink_attr {
/* don't change the order or add anything between, this is ABI! */
DEVLINK_ATTR_UNSPEC,
......@@ -238,6 +253,15 @@ enum devlink_attr {
DEVLINK_ATTR_PORT_NUMBER, /* u32 */
DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER, /* u32 */
DEVLINK_ATTR_PARAM, /* nested */
DEVLINK_ATTR_PARAM_NAME, /* string */
DEVLINK_ATTR_PARAM_GENERIC, /* flag */
DEVLINK_ATTR_PARAM_TYPE, /* u8 */
DEVLINK_ATTR_PARAM_VALUES_LIST, /* nested */
DEVLINK_ATTR_PARAM_VALUE, /* nested */
DEVLINK_ATTR_PARAM_VALUE_DATA, /* dynamic */
DEVLINK_ATTR_PARAM_VALUE_CMODE, /* u8 */
/* add new attributes above here, update the policy in devlink.c */
__DEVLINK_ATTR_MAX,
......
......@@ -2604,6 +2604,500 @@ static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
return devlink->ops->reload(devlink, info->extack);
}
static const struct devlink_param devlink_param_generic[] = {
{
.id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
.name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
.type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
},
{
.id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
.name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
.type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
},
{
.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
.name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
.type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
},
};
static int devlink_param_generic_verify(const struct devlink_param *param)
{
/* verify it match generic parameter by id and name */
if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
return -EINVAL;
if (strcmp(param->name, devlink_param_generic[param->id].name))
return -ENOENT;
WARN_ON(param->type != devlink_param_generic[param->id].type);
return 0;
}
static int devlink_param_driver_verify(const struct devlink_param *param)
{
int i;
if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
return -EINVAL;
/* verify no such name in generic params */
for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
if (!strcmp(param->name, devlink_param_generic[i].name))
return -EEXIST;
return 0;
}
static struct devlink_param_item *
devlink_param_find_by_name(struct list_head *param_list,
const char *param_name)
{
struct devlink_param_item *param_item;
list_for_each_entry(param_item, param_list, list)
if (!strcmp(param_item->param->name, param_name))
return param_item;
return NULL;
}
static struct devlink_param_item *
devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
{
struct devlink_param_item *param_item;
list_for_each_entry(param_item, param_list, list)
if (param_item->param->id == param_id)
return param_item;
return NULL;
}
static bool
devlink_param_cmode_is_supported(const struct devlink_param *param,
enum devlink_param_cmode cmode)
{
return test_bit(cmode, &param->supported_cmodes);
}
static int devlink_param_get(struct devlink *devlink,
const struct devlink_param *param,
struct devlink_param_gset_ctx *ctx)
{
if (!param->get)
return -EOPNOTSUPP;
return param->get(devlink, param->id, ctx);
}
static int devlink_param_set(struct devlink *devlink,
const struct devlink_param *param,
struct devlink_param_gset_ctx *ctx)
{
if (!param->set)
return -EOPNOTSUPP;
return param->set(devlink, param->id, ctx);
}
static int
devlink_param_type_to_nla_type(enum devlink_param_type param_type)
{
switch (param_type) {
case DEVLINK_PARAM_TYPE_U8:
return NLA_U8;
case DEVLINK_PARAM_TYPE_U16:
return NLA_U16;
case DEVLINK_PARAM_TYPE_U32:
return NLA_U32;
case DEVLINK_PARAM_TYPE_STRING:
return NLA_STRING;
case DEVLINK_PARAM_TYPE_BOOL:
return NLA_FLAG;
default:
return -EINVAL;
}
}
static int
devlink_nl_param_value_fill_one(struct sk_buff *msg,
enum devlink_param_type type,
enum devlink_param_cmode cmode,
union devlink_param_value val)
{
struct nlattr *param_value_attr;
param_value_attr = nla_nest_start(msg, DEVLINK_ATTR_PARAM_VALUE);
if (!param_value_attr)
goto nla_put_failure;
if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
goto value_nest_cancel;
switch (type) {
case DEVLINK_PARAM_TYPE_U8:
if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
goto value_nest_cancel;
break;
case DEVLINK_PARAM_TYPE_U16:
if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
goto value_nest_cancel;
break;
case DEVLINK_PARAM_TYPE_U32:
if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
goto value_nest_cancel;
break;
case DEVLINK_PARAM_TYPE_STRING:
if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
val.vstr))
goto value_nest_cancel;
break;
case DEVLINK_PARAM_TYPE_BOOL:
if (val.vbool &&
nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
goto value_nest_cancel;
break;
}
nla_nest_end(msg, param_value_attr);
return 0;
value_nest_cancel:
nla_nest_cancel(msg, param_value_attr);
nla_put_failure:
return -EMSGSIZE;
}
static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
struct devlink_param_item *param_item,
enum devlink_command cmd,
u32 portid, u32 seq, int flags)
{
union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
const struct devlink_param *param = param_item->param;
struct devlink_param_gset_ctx ctx;
struct nlattr *param_values_list;
struct nlattr *param_attr;
int nla_type;
void *hdr;
int err;
int i;
/* Get value from driver part to driverinit configuration mode */
for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
if (!devlink_param_cmode_is_supported(param, i))
continue;
if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
if (!param_item->driverinit_value_valid)
return -EOPNOTSUPP;
param_value[i] = param_item->driverinit_value;
} else {
ctx.cmode = i;
err = devlink_param_get(devlink, param, &ctx);
if (err)
return err;
param_value[i] = ctx.val;
}
}
hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
if (!hdr)
return -EMSGSIZE;
if (devlink_nl_put_handle(msg, devlink))
goto genlmsg_cancel;
param_attr = nla_nest_start(msg, DEVLINK_ATTR_PARAM);
if (!param_attr)
goto genlmsg_cancel;
if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
goto param_nest_cancel;
if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
goto param_nest_cancel;
nla_type = devlink_param_type_to_nla_type(param->type);
if (nla_type < 0)
goto param_nest_cancel;
if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
goto param_nest_cancel;
param_values_list = nla_nest_start(msg, DEVLINK_ATTR_PARAM_VALUES_LIST);
if (!param_values_list)
goto param_nest_cancel;
for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
if (!devlink_param_cmode_is_supported(param, i))
continue;
err = devlink_nl_param_value_fill_one(msg, param->type,
i, param_value[i]);
if (err)
goto values_list_nest_cancel;
}
nla_nest_end(msg, param_values_list);
nla_nest_end(msg, param_attr);
genlmsg_end(msg, hdr);
return 0;
values_list_nest_cancel:
nla_nest_end(msg, param_values_list);
param_nest_cancel:
nla_nest_cancel(msg, param_attr);
genlmsg_cancel:
genlmsg_cancel(msg, hdr);
return -EMSGSIZE;
}
static void devlink_param_notify(struct devlink *devlink,
struct devlink_param_item *param_item,
enum devlink_command cmd)
{
struct sk_buff *msg;
int err;
WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL);
msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
if (!msg)
return;
err = devlink_nl_param_fill(msg, devlink, param_item, cmd, 0, 0, 0);
if (err) {
nlmsg_free(msg);
return;
}
genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
}
static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
struct netlink_callback *cb)
{
struct devlink_param_item *param_item;
struct devlink *devlink;
int start = cb->args[0];
int idx = 0;
int err;
mutex_lock(&devlink_mutex);
list_for_each_entry(devlink, &devlink_list, list) {
if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
continue;
mutex_lock(&devlink->lock);
list_for_each_entry(param_item, &devlink->param_list, list) {
if (idx < start) {
idx++;
continue;
}
err = devlink_nl_param_fill(msg, devlink, param_item,
DEVLINK_CMD_PARAM_GET,
NETLINK_CB(cb->skb).portid,
cb->nlh->nlmsg_seq,
NLM_F_MULTI);
if (err) {
mutex_unlock(&devlink->lock);
goto out;
}
idx++;
}
mutex_unlock(&devlink->lock);
}
out:
mutex_unlock(&devlink_mutex);
cb->args[0] = idx;
return msg->len;
}
static int
devlink_param_type_get_from_info(struct genl_info *info,
enum devlink_param_type *param_type)
{
if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
return -EINVAL;
switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
case NLA_U8:
*param_type = DEVLINK_PARAM_TYPE_U8;
break;
case NLA_U16:
*param_type = DEVLINK_PARAM_TYPE_U16;
break;
case NLA_U32:
*param_type = DEVLINK_PARAM_TYPE_U32;
break;
case NLA_STRING:
*param_type = DEVLINK_PARAM_TYPE_STRING;
break;
case NLA_FLAG:
*param_type = DEVLINK_PARAM_TYPE_BOOL;
break;
default:
return -EINVAL;
}
return 0;
}
static int
devlink_param_value_get_from_info(const struct devlink_param *param,
struct genl_info *info,
union devlink_param_value *value)
{
if (param->type != DEVLINK_PARAM_TYPE_BOOL &&
!info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA])
return -EINVAL;
switch (param->type) {
case DEVLINK_PARAM_TYPE_U8:
value->vu8 = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA]);
break;
case DEVLINK_PARAM_TYPE_U16:
value->vu16 = nla_get_u16(info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA]);
break;
case DEVLINK_PARAM_TYPE_U32:
value->vu32 = nla_get_u32(info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA]);
break;
case DEVLINK_PARAM_TYPE_STRING:
if (nla_len(info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA]) >
DEVLINK_PARAM_MAX_STRING_VALUE)
return -EINVAL;
value->vstr = nla_data(info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA]);
break;
case DEVLINK_PARAM_TYPE_BOOL:
value->vbool = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA] ?
true : false;
break;
}
return 0;
}
static struct devlink_param_item *
devlink_param_get_from_info(struct devlink *devlink,
struct genl_info *info)
{
char *param_name;
if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
return NULL;
param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
return devlink_param_find_by_name(&devlink->param_list, param_name);
}
static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
struct genl_info *info)
{
struct devlink *devlink = info->user_ptr[0];
struct devlink_param_item *param_item;
struct sk_buff *msg;
int err;
param_item = devlink_param_get_from_info(devlink, info);
if (!param_item)
return -EINVAL;
msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
if (!msg)
return -ENOMEM;
err = devlink_nl_param_fill(msg, devlink, param_item,
DEVLINK_CMD_PARAM_GET,
info->snd_portid, info->snd_seq, 0);
if (err) {
nlmsg_free(msg);
return err;
}
return genlmsg_reply(msg, info);
}
static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
struct genl_info *info)
{
struct devlink *devlink = info->user_ptr[0];
enum devlink_param_type param_type;
struct devlink_param_gset_ctx ctx;
enum devlink_param_cmode cmode;
struct devlink_param_item *param_item;
const struct devlink_param *param;
union devlink_param_value value;
int err = 0;
param_item = devlink_param_get_from_info(devlink, info);
if (!param_item)
return -EINVAL;
param = param_item->param;
err = devlink_param_type_get_from_info(info, &param_type);
if (err)
return err;
if (param_type != param->type)
return -EINVAL;
err = devlink_param_value_get_from_info(param, info, &value);
if (err)
return err;
if (param->validate) {
err = param->validate(devlink, param->id, value, info->extack);
if (err)
return err;
}
if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
return -EINVAL;
cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
if (!devlink_param_cmode_is_supported(param, cmode))
return -EOPNOTSUPP;
if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
param_item->driverinit_value = value;
param_item->driverinit_value_valid = true;
} else {
if (!param->set)
return -EOPNOTSUPP;
ctx.val = value;
ctx.cmode = cmode;
err = devlink_param_set(devlink, param, &ctx);
if (err)
return err;
}
devlink_param_notify(devlink, param_item, DEVLINK_CMD_PARAM_NEW);
return 0;
}
static int devlink_param_register_one(struct devlink *devlink,
const struct devlink_param *param)
{
struct devlink_param_item *param_item;
if (devlink_param_find_by_name(&devlink->param_list,
param->name))
return -EEXIST;
if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
WARN_ON(param->get || param->set);
else
WARN_ON(!param->get || !param->set);
param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
if (!param_item)
return -ENOMEM;
param_item->param = param;
list_add_tail(&param_item->list, &devlink->param_list);
devlink_param_notify(devlink, param_item, DEVLINK_CMD_PARAM_NEW);
return 0;
}
static void devlink_param_unregister_one(struct devlink *devlink,
const struct devlink_param *param)
{
struct devlink_param_item *param_item;
param_item = devlink_param_find_by_name(&devlink->param_list,
param->name);
WARN_ON(!param_item);
devlink_param_notify(devlink, param_item, DEVLINK_CMD_PARAM_DEL);
list_del(&param_item->list);
kfree(param_item);
}
static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
......@@ -2624,6 +3118,9 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
[DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
[DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
[DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
[DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
[DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
};
static const struct genl_ops devlink_nl_ops[] = {
......@@ -2807,6 +3304,21 @@ static const struct genl_ops devlink_nl_ops[] = {
.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK |
DEVLINK_NL_FLAG_NO_LOCK,
},
{
.cmd = DEVLINK_CMD_PARAM_GET,
.doit = devlink_nl_cmd_param_get_doit,
.dumpit = devlink_nl_cmd_param_get_dumpit,
.policy = devlink_nl_policy,
.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
/* can be retrieved by unprivileged users */
},
{
.cmd = DEVLINK_CMD_PARAM_SET,
.doit = devlink_nl_cmd_param_set_doit,
.policy = devlink_nl_policy,
.flags = GENL_ADMIN_PERM,
.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK,
},
};
static struct genl_family devlink_nl_family __ro_after_init = {
......@@ -2845,6 +3357,7 @@ struct devlink *devlink_alloc(const struct devlink_ops *ops, size_t priv_size)
INIT_LIST_HEAD(&devlink->sb_list);
INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
INIT_LIST_HEAD(&devlink->resource_list);
INIT_LIST_HEAD(&devlink->param_list);
mutex_init(&devlink->lock);
return devlink;
}
......@@ -3434,6 +3947,168 @@ void devlink_resource_occ_get_unregister(struct devlink *devlink,
}
EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
/**
* devlink_params_register - register configuration parameters
*
* @devlink: devlink
* @params: configuration parameters array
* @params_count: number of parameters provided
*
* Register the configuration parameters supported by the driver.
*/
int devlink_params_register(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count)
{
const struct devlink_param *param = params;
int i;
int err;
mutex_lock(&devlink->lock);
for (i = 0; i < params_count; i++, param++) {
if (!param || !param->name || !param->supported_cmodes) {
err = -EINVAL;
goto rollback;
}
if (param->generic) {
err = devlink_param_generic_verify(param);
if (err)
goto rollback;
} else {
err = devlink_param_driver_verify(param);
if (err)
goto rollback;
}
err = devlink_param_register_one(devlink, param);
if (err)
goto rollback;
}
mutex_unlock(&devlink->lock);
return 0;
rollback:
if (!i)
goto unlock;
for (param--; i > 0; i--, param--)
devlink_param_unregister_one(devlink, param);
unlock:
mutex_unlock(&devlink->lock);
return err;
}
EXPORT_SYMBOL_GPL(devlink_params_register);
/**
* devlink_params_unregister - unregister configuration parameters
* @devlink: devlink
* @params: configuration parameters to unregister
* @params_count: number of parameters provided
*/
void devlink_params_unregister(struct devlink *devlink,
const struct devlink_param *params,
size_t params_count)
{
const struct devlink_param *param = params;
int i;
mutex_lock(&devlink->lock);
for (i = 0; i < params_count; i++, param++)
devlink_param_unregister_one(devlink, param);
mutex_unlock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devlink_params_unregister);
/**
* devlink_param_driverinit_value_get - get configuration parameter
* value for driver initializing
*
* @devlink: devlink
* @param_id: parameter ID
* @init_val: value of parameter in driverinit configuration mode
*
* This function should be used by the driver to get driverinit
* configuration for initialization after reload command.
*/
int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
union devlink_param_value *init_val)
{
struct devlink_param_item *param_item;
if (!devlink->ops || !devlink->ops->reload)
return -EOPNOTSUPP;
param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
if (!param_item)
return -EINVAL;
if (!param_item->driverinit_value_valid ||
!devlink_param_cmode_is_supported(param_item->param,
DEVLINK_PARAM_CMODE_DRIVERINIT))
return -EOPNOTSUPP;
*init_val = param_item->driverinit_value;
return 0;
}
EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
/**
* devlink_param_driverinit_value_set - set value of configuration
* parameter for driverinit
* configuration mode
*
* @devlink: devlink
* @param_id: parameter ID
* @init_val: value of parameter to set for driverinit configuration mode
*
* This function should be used by the driver to set driverinit
* configuration mode default value.
*/
int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
union devlink_param_value init_val)
{
struct devlink_param_item *param_item;
param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
if (!param_item)
return -EINVAL;
if (!devlink_param_cmode_is_supported(param_item->param,
DEVLINK_PARAM_CMODE_DRIVERINIT))
return -EOPNOTSUPP;
param_item->driverinit_value = init_val;
param_item->driverinit_value_valid = true;
devlink_param_notify(devlink, param_item, DEVLINK_CMD_PARAM_NEW);
return 0;
}
EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
/**
* devlink_param_value_changed - notify devlink on a parameter's value
* change. Should be called by the driver
* right after the change.
*
* @devlink: devlink
* @param_id: parameter ID
*
* This function should be used by the driver to notify devlink on value
* change, excluding driverinit configuration mode.
* For driverinit configuration mode driver should use the function
* devlink_param_driverinit_value_set() instead.
*/
void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
{
struct devlink_param_item *param_item;
param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
WARN_ON(!param_item);
devlink_param_notify(devlink, param_item, DEVLINK_CMD_PARAM_NEW);
}
EXPORT_SYMBOL_GPL(devlink_param_value_changed);
static int __init devlink_module_init(void)
{
return genl_register_family(&devlink_nl_family);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment