Commit 8947c390 authored by Jakub Kicinski's avatar Jakub Kicinski

Merge branch 'mlxsw-add-spectrum-4-support'

Ido Schimmel says:

====================
mlxsw: Add Spectrum-4 support

This patchset adds Spectrum-4 support in mlxsw. It builds on top of a
previous patchset merged in commit 10184da9 ("Merge branch
'mlxsw-Spectrum-4-prep'") and makes two additional changes before adding
Spectrum-4 support.

Patchset overview:

Patches #1-#2 add a few Spectrum-4 specific variants of existing ACL
keys. The new variants are needed because the size of certain key
elements (e.g., local port) was increased in Spectrum-4.

Patches #3-#6 are preparations.

Patch #7 implements the Spectrum-4 variant of the Bloom filter hash
function. The Bloom filter is used to optimize ACL lookups by
potentially skipping certain lookups if they are guaranteed not to
match. See additional info in merge commit ae6750e0 ("Merge branch
'mlxsw-spectrum_acl-Add-Bloom-filter-support'").

Patch #8 finally adds Spectrum-4 support.
====================

Link: https://lore.kernel.org/r/20220106160652.821176-1-idosch@nvidia.comSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 42379b95 47354021
......@@ -66,7 +66,7 @@ config MLXSW_SPECTRUM
default m
help
This driver supports Mellanox Technologies
Spectrum/Spectrum-2/Spectrum-3 Ethernet Switch ASICs.
Spectrum/Spectrum-2/Spectrum-3/Spectrum-4 Ethernet Switch ASICs.
To compile this driver as a module, choose M here: the
module will be called mlxsw_spectrum.
......
......@@ -32,8 +32,8 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_8_10, 0x18, 17, 3),
MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_0_7, 0x18, 20, 8),
MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 3),
MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 20, 8),
MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4),
MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4),
MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4),
......
......@@ -33,8 +33,8 @@ enum mlxsw_afk_element {
MLXSW_AFK_ELEMENT_IP_TTL_,
MLXSW_AFK_ELEMENT_IP_ECN,
MLXSW_AFK_ELEMENT_IP_DSCP,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_8_10,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_0_7,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB,
MLXSW_AFK_ELEMENT_MAX,
};
......
......@@ -9,6 +9,7 @@
#define PCI_DEVICE_ID_MELLANOX_SPECTRUM 0xcb84
#define PCI_DEVICE_ID_MELLANOX_SPECTRUM2 0xcf6c
#define PCI_DEVICE_ID_MELLANOX_SPECTRUM3 0xcf70
#define PCI_DEVICE_ID_MELLANOX_SPECTRUM4 0xcf80
#if IS_ENABLED(CONFIG_MLXSW_PCI)
......
......@@ -3914,6 +3914,7 @@ MLXSW_ITEM32(reg, qeec, max_shaper_bs, 0x1C, 0, 6);
#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP1 5
#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP2 11
#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP3 11
#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP4 11
static inline void mlxsw_reg_qeec_pack(char *payload, u16 local_port,
enum mlxsw_reg_qeec_hr hr, u8 index,
......
......@@ -95,6 +95,7 @@ static const struct mlxsw_fw_rev mlxsw_sp3_fw_rev = {
static const char mlxsw_sp1_driver_name[] = "mlxsw_spectrum";
static const char mlxsw_sp2_driver_name[] = "mlxsw_spectrum2";
static const char mlxsw_sp3_driver_name[] = "mlxsw_spectrum3";
static const char mlxsw_sp4_driver_name[] = "mlxsw_spectrum4";
static const unsigned char mlxsw_sp1_mac_mask[ETH_ALEN] = {
0xff, 0xff, 0xff, 0xff, 0xfc, 0x00
......@@ -3155,6 +3156,7 @@ static int mlxsw_sp2_init(struct mlxsw_core *mlxsw_core,
mlxsw_sp->mr_tcam_ops = &mlxsw_sp2_mr_tcam_ops;
mlxsw_sp->acl_rulei_ops = &mlxsw_sp2_acl_rulei_ops;
mlxsw_sp->acl_tcam_ops = &mlxsw_sp2_acl_tcam_ops;
mlxsw_sp->acl_bf_ops = &mlxsw_sp2_acl_bf_ops;
mlxsw_sp->nve_ops_arr = mlxsw_sp2_nve_ops_arr;
mlxsw_sp->mac_mask = mlxsw_sp2_mac_mask;
mlxsw_sp->sb_vals = &mlxsw_sp2_sb_vals;
......@@ -3184,6 +3186,7 @@ static int mlxsw_sp3_init(struct mlxsw_core *mlxsw_core,
mlxsw_sp->mr_tcam_ops = &mlxsw_sp2_mr_tcam_ops;
mlxsw_sp->acl_rulei_ops = &mlxsw_sp2_acl_rulei_ops;
mlxsw_sp->acl_tcam_ops = &mlxsw_sp2_acl_tcam_ops;
mlxsw_sp->acl_bf_ops = &mlxsw_sp2_acl_bf_ops;
mlxsw_sp->nve_ops_arr = mlxsw_sp2_nve_ops_arr;
mlxsw_sp->mac_mask = mlxsw_sp2_mac_mask;
mlxsw_sp->sb_vals = &mlxsw_sp2_sb_vals;
......@@ -3200,6 +3203,36 @@ static int mlxsw_sp3_init(struct mlxsw_core *mlxsw_core,
return mlxsw_sp_init(mlxsw_core, mlxsw_bus_info, extack);
}
static int mlxsw_sp4_init(struct mlxsw_core *mlxsw_core,
const struct mlxsw_bus_info *mlxsw_bus_info,
struct netlink_ext_ack *extack)
{
struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
mlxsw_sp->switchdev_ops = &mlxsw_sp2_switchdev_ops;
mlxsw_sp->kvdl_ops = &mlxsw_sp2_kvdl_ops;
mlxsw_sp->afa_ops = &mlxsw_sp2_act_afa_ops;
mlxsw_sp->afk_ops = &mlxsw_sp4_afk_ops;
mlxsw_sp->mr_tcam_ops = &mlxsw_sp2_mr_tcam_ops;
mlxsw_sp->acl_rulei_ops = &mlxsw_sp2_acl_rulei_ops;
mlxsw_sp->acl_tcam_ops = &mlxsw_sp2_acl_tcam_ops;
mlxsw_sp->acl_bf_ops = &mlxsw_sp4_acl_bf_ops;
mlxsw_sp->nve_ops_arr = mlxsw_sp2_nve_ops_arr;
mlxsw_sp->mac_mask = mlxsw_sp2_mac_mask;
mlxsw_sp->sb_vals = &mlxsw_sp2_sb_vals;
mlxsw_sp->sb_ops = &mlxsw_sp3_sb_ops;
mlxsw_sp->port_type_speed_ops = &mlxsw_sp2_port_type_speed_ops;
mlxsw_sp->ptp_ops = &mlxsw_sp2_ptp_ops;
mlxsw_sp->span_ops = &mlxsw_sp3_span_ops;
mlxsw_sp->policer_core_ops = &mlxsw_sp2_policer_core_ops;
mlxsw_sp->trap_ops = &mlxsw_sp2_trap_ops;
mlxsw_sp->mall_ops = &mlxsw_sp2_mall_ops;
mlxsw_sp->router_ops = &mlxsw_sp2_router_ops;
mlxsw_sp->lowest_shaper_bs = MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP4;
return mlxsw_sp_init(mlxsw_core, mlxsw_bus_info, extack);
}
static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
{
struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
......@@ -3759,6 +3792,45 @@ static struct mlxsw_driver mlxsw_sp3_driver = {
.temp_warn_enabled = true,
};
static struct mlxsw_driver mlxsw_sp4_driver = {
.kind = mlxsw_sp4_driver_name,
.priv_size = sizeof(struct mlxsw_sp),
.init = mlxsw_sp4_init,
.fini = mlxsw_sp_fini,
.basic_trap_groups_set = mlxsw_sp_basic_trap_groups_set,
.port_split = mlxsw_sp_port_split,
.port_unsplit = mlxsw_sp_port_unsplit,
.sb_pool_get = mlxsw_sp_sb_pool_get,
.sb_pool_set = mlxsw_sp_sb_pool_set,
.sb_port_pool_get = mlxsw_sp_sb_port_pool_get,
.sb_port_pool_set = mlxsw_sp_sb_port_pool_set,
.sb_tc_pool_bind_get = mlxsw_sp_sb_tc_pool_bind_get,
.sb_tc_pool_bind_set = mlxsw_sp_sb_tc_pool_bind_set,
.sb_occ_snapshot = mlxsw_sp_sb_occ_snapshot,
.sb_occ_max_clear = mlxsw_sp_sb_occ_max_clear,
.sb_occ_port_pool_get = mlxsw_sp_sb_occ_port_pool_get,
.sb_occ_tc_port_bind_get = mlxsw_sp_sb_occ_tc_port_bind_get,
.trap_init = mlxsw_sp_trap_init,
.trap_fini = mlxsw_sp_trap_fini,
.trap_action_set = mlxsw_sp_trap_action_set,
.trap_group_init = mlxsw_sp_trap_group_init,
.trap_group_set = mlxsw_sp_trap_group_set,
.trap_policer_init = mlxsw_sp_trap_policer_init,
.trap_policer_fini = mlxsw_sp_trap_policer_fini,
.trap_policer_set = mlxsw_sp_trap_policer_set,
.trap_policer_counter_get = mlxsw_sp_trap_policer_counter_get,
.txhdr_construct = mlxsw_sp_txhdr_construct,
.resources_register = mlxsw_sp2_resources_register,
.params_register = mlxsw_sp2_params_register,
.params_unregister = mlxsw_sp2_params_unregister,
.ptp_transmitted = mlxsw_sp_ptp_transmitted,
.txhdr_len = MLXSW_TXHDR_LEN,
.profile = &mlxsw_sp2_config_profile,
.res_query_enabled = true,
.fw_fatal_enabled = true,
.temp_warn_enabled = true,
};
bool mlxsw_sp_port_dev_check(const struct net_device *dev)
{
return dev->netdev_ops == &mlxsw_sp_port_netdev_ops;
......@@ -4926,6 +4998,16 @@ static struct pci_driver mlxsw_sp3_pci_driver = {
.id_table = mlxsw_sp3_pci_id_table,
};
static const struct pci_device_id mlxsw_sp4_pci_id_table[] = {
{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM4), 0},
{0, },
};
static struct pci_driver mlxsw_sp4_pci_driver = {
.name = mlxsw_sp4_driver_name,
.id_table = mlxsw_sp4_pci_id_table,
};
static int __init mlxsw_sp_module_init(void)
{
int err;
......@@ -4945,6 +5027,10 @@ static int __init mlxsw_sp_module_init(void)
if (err)
goto err_sp3_core_driver_register;
err = mlxsw_core_driver_register(&mlxsw_sp4_driver);
if (err)
goto err_sp4_core_driver_register;
err = mlxsw_pci_driver_register(&mlxsw_sp1_pci_driver);
if (err)
goto err_sp1_pci_driver_register;
......@@ -4957,13 +5043,21 @@ static int __init mlxsw_sp_module_init(void)
if (err)
goto err_sp3_pci_driver_register;
err = mlxsw_pci_driver_register(&mlxsw_sp4_pci_driver);
if (err)
goto err_sp4_pci_driver_register;
return 0;
err_sp4_pci_driver_register:
mlxsw_pci_driver_unregister(&mlxsw_sp3_pci_driver);
err_sp3_pci_driver_register:
mlxsw_pci_driver_unregister(&mlxsw_sp2_pci_driver);
err_sp2_pci_driver_register:
mlxsw_pci_driver_unregister(&mlxsw_sp1_pci_driver);
err_sp1_pci_driver_register:
mlxsw_core_driver_unregister(&mlxsw_sp4_driver);
err_sp4_core_driver_register:
mlxsw_core_driver_unregister(&mlxsw_sp3_driver);
err_sp3_core_driver_register:
mlxsw_core_driver_unregister(&mlxsw_sp2_driver);
......@@ -4977,9 +5071,11 @@ static int __init mlxsw_sp_module_init(void)
static void __exit mlxsw_sp_module_exit(void)
{
mlxsw_pci_driver_unregister(&mlxsw_sp4_pci_driver);
mlxsw_pci_driver_unregister(&mlxsw_sp3_pci_driver);
mlxsw_pci_driver_unregister(&mlxsw_sp2_pci_driver);
mlxsw_pci_driver_unregister(&mlxsw_sp1_pci_driver);
mlxsw_core_driver_unregister(&mlxsw_sp4_driver);
mlxsw_core_driver_unregister(&mlxsw_sp3_driver);
mlxsw_core_driver_unregister(&mlxsw_sp2_driver);
mlxsw_core_driver_unregister(&mlxsw_sp1_driver);
......@@ -4996,6 +5092,7 @@ MODULE_DESCRIPTION("Mellanox Spectrum driver");
MODULE_DEVICE_TABLE(pci, mlxsw_sp1_pci_id_table);
MODULE_DEVICE_TABLE(pci, mlxsw_sp2_pci_id_table);
MODULE_DEVICE_TABLE(pci, mlxsw_sp3_pci_id_table);
MODULE_DEVICE_TABLE(pci, mlxsw_sp4_pci_id_table);
MODULE_FIRMWARE(MLXSW_SP1_FW_FILENAME);
MODULE_FIRMWARE(MLXSW_SP2_FW_FILENAME);
MODULE_FIRMWARE(MLXSW_SP3_FW_FILENAME);
......@@ -190,6 +190,7 @@ struct mlxsw_sp {
const struct mlxsw_sp_mr_tcam_ops *mr_tcam_ops;
const struct mlxsw_sp_acl_rulei_ops *acl_rulei_ops;
const struct mlxsw_sp_acl_tcam_ops *acl_tcam_ops;
const struct mlxsw_sp_acl_bf_ops *acl_bf_ops;
const struct mlxsw_sp_nve_ops **nve_ops_arr;
const struct mlxsw_sp_sb_vals *sb_vals;
const struct mlxsw_sp_sb_ops *sb_ops;
......@@ -1106,6 +1107,11 @@ extern const struct mlxsw_afa_ops mlxsw_sp2_act_afa_ops;
/* spectrum_acl_flex_keys.c */
extern const struct mlxsw_afk_ops mlxsw_sp1_afk_ops;
extern const struct mlxsw_afk_ops mlxsw_sp2_afk_ops;
extern const struct mlxsw_afk_ops mlxsw_sp4_afk_ops;
/* spectrum_acl_bloom_filter.c */
extern const struct mlxsw_sp_acl_bf_ops mlxsw_sp2_acl_bf_ops;
extern const struct mlxsw_sp_acl_bf_ops mlxsw_sp4_acl_bf_ops;
/* spectrum_matchall.c */
struct mlxsw_sp_mall_ops {
......
......@@ -45,8 +45,8 @@ static int mlxsw_sp2_mr_tcam_bind_group(struct mlxsw_sp *mlxsw_sp,
}
static const enum mlxsw_afk_element mlxsw_sp2_mr_tcam_usage_ipv4[] = {
MLXSW_AFK_ELEMENT_VIRT_ROUTER_8_10,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_0_7,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB,
MLXSW_AFK_ELEMENT_SRC_IP_0_31,
MLXSW_AFK_ELEMENT_DST_IP_0_31,
};
......@@ -89,8 +89,8 @@ static void mlxsw_sp2_mr_tcam_ipv4_fini(struct mlxsw_sp2_mr_tcam *mr_tcam)
}
static const enum mlxsw_afk_element mlxsw_sp2_mr_tcam_usage_ipv6[] = {
MLXSW_AFK_ELEMENT_VIRT_ROUTER_8_10,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_0_7,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB,
MLXSW_AFK_ELEMENT_SRC_IP_96_127,
MLXSW_AFK_ELEMENT_SRC_IP_64_95,
MLXSW_AFK_ELEMENT_SRC_IP_32_63,
......@@ -189,10 +189,10 @@ mlxsw_sp2_mr_tcam_rule_parse(struct mlxsw_sp_acl_rule *rule,
rulei = mlxsw_sp_acl_rule_rulei(rule);
rulei->priority = priority;
mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_VIRT_ROUTER_0_7,
mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_VIRT_ROUTER_LSB,
key->vrid, GENMASK(7, 0));
mlxsw_sp_acl_rulei_keymask_u32(rulei,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_8_10,
MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
key->vrid >> 8, GENMASK(2, 0));
switch (key->proto) {
case MLXSW_SP_L3_PROTO_IPV4:
......
......@@ -17,9 +17,9 @@ struct mlxsw_sp_acl_bf {
};
/* Bloom filter uses a crc-16 hash over chunks of data which contain 4 key
* blocks, eRP ID and region ID. In Spectrum-2, region key is combined of up to
* 12 key blocks, so there can be up to 3 chunks in the Bloom filter key,
* depending on the actual number of key blocks used in the region.
* blocks, eRP ID and region ID. In Spectrum-2 and above, region key is combined
* of up to 12 key blocks, so there can be up to 3 chunks in the Bloom filter
* key, depending on the actual number of key blocks used in the region.
* The layout of the Bloom filter key is as follows:
*
* +-------------------------+------------------------+------------------------+
......@@ -27,7 +27,9 @@ struct mlxsw_sp_acl_bf {
* +-------------------------+------------------------+------------------------+
*/
#define MLXSW_BLOOM_KEY_CHUNKS 3
#define MLXSW_BLOOM_KEY_LEN 69
/* Spectrum-2 and Spectrum-3 chunks */
#define MLXSW_SP2_BLOOM_KEY_LEN 69
/* Each chunk size is 23 bytes. 18 bytes of it contain 4 key blocks, each is
* 36 bits, 2 bytes which hold eRP ID and region ID, and 3 bytes of zero
......@@ -42,31 +44,21 @@ struct mlxsw_sp_acl_bf {
* | 0 | region ID | eRP ID | 4 Key blocks (18 Bytes) |
* +---------+-----------+----------+-----------------------------------+
*/
#define MLXSW_BLOOM_CHUNK_PAD_BYTES 3
#define MLXSW_BLOOM_CHUNK_KEY_BYTES 18
#define MLXSW_BLOOM_KEY_CHUNK_BYTES 23
#define MLXSW_SP2_BLOOM_CHUNK_PAD_BYTES 3
#define MLXSW_SP2_BLOOM_CHUNK_KEY_BYTES 18
#define MLXSW_SP2_BLOOM_KEY_CHUNK_BYTES 23
/* The offset of the key block within a chunk is 5 bytes as it comes after
* 3 bytes of zero padding and 16 bits of region ID and eRP ID.
*/
#define MLXSW_BLOOM_CHUNK_KEY_OFFSET 5
#define MLXSW_SP2_BLOOM_CHUNK_KEY_OFFSET 5
/* Each chunk contains 4 key blocks. Chunk 2 uses key blocks 11-8,
* and we need to populate it with 4 key blocks copied from the entry encoded
* key. Since the encoded key contains a padding, key block 11 starts at offset
* 2. block 7 that is used in chunk 1 starts at offset 20 as 4 key blocks take
* 18 bytes.
* This array defines key offsets for easy access when copying key blocks from
* entry key to Bloom filter chunk.
*/
static const u8 chunk_key_offsets[MLXSW_BLOOM_KEY_CHUNKS] = {2, 20, 38};
/* This table is just the CRC of each possible byte. It is
* computed, Msbit first, for the Bloom filter polynomial
* which is 0x8529 (1 + x^3 + x^5 + x^8 + x^10 + x^15 and
/* This table is just the CRC of each possible byte which is used for
* Spectrum-{2-3}. It is computed, Msbit first, for the Bloom filter
* polynomial which is 0x8529 (1 + x^3 + x^5 + x^8 + x^10 + x^15 and
* the implicit x^16).
*/
static const u16 mlxsw_sp_acl_bf_crc_tab[256] = {
static const u16 mlxsw_sp2_acl_bf_crc16_tab[256] = {
0x0000, 0x8529, 0x8f7b, 0x0a52, 0x9bdf, 0x1ef6, 0x14a4, 0x918d,
0xb297, 0x37be, 0x3dec, 0xb8c5, 0x2948, 0xac61, 0xa633, 0x231a,
0xe007, 0x652e, 0x6f7c, 0xea55, 0x7bd8, 0xfef1, 0xf4a3, 0x718a,
......@@ -101,24 +93,146 @@ static const u16 mlxsw_sp_acl_bf_crc_tab[256] = {
0x0c4c, 0x8965, 0x8337, 0x061e, 0x9793, 0x12ba, 0x18e8, 0x9dc1,
};
static u16 mlxsw_sp_acl_bf_crc_byte(u16 crc, u8 c)
/* Spectrum-4 chunks */
#define MLXSW_SP4_BLOOM_KEY_LEN 60
/* In Spectrum-4, there is no padding. Each chunk size is 20 bytes.
* 18 bytes of it contain 4 key blocks, each is 36 bits, and 2 bytes which hold
* eRP ID and region ID.
* The layout of each chunk is as follows:
*
* +----------------------+-----------------------------------+
* | 2 bytes | 18 bytes |
* +-----------+----------+-----------------------------------+
* | 157:148 | 147:144 | 143:0 |
* +---------+-----------+----------+-------------------------+
* | region ID | eRP ID | 4 Key blocks (18 Bytes) |
* +-----------+----------+-----------------------------------+
*/
#define MLXSW_SP4_BLOOM_CHUNK_PAD_BYTES 0
#define MLXSW_SP4_BLOOM_CHUNK_KEY_BYTES 18
#define MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES 20
/* The offset of the key block within a chunk is 2 bytes as it comes after
* 16 bits of region ID and eRP ID.
*/
#define MLXSW_SP4_BLOOM_CHUNK_KEY_OFFSET 2
/* For Spectrum-4, two hash functions are used, CRC-10 and CRC-6 based.
* The result is combination of the two calculations -
* 6 bit column are MSB (result of CRC-6),
* 10 bit row are LSB (result of CRC-10).
*/
/* This table is just the CRC of each possible byte which is used for
* Spectrum-4. It is computed, Msbit first, for the Bloom filter
* polynomial which is 0x1b (1 + x^1 + x^3 + x^4 and the implicit x^10).
*/
static const u16 mlxsw_sp4_acl_bf_crc10_tab[256] = {
0x0000, 0x001b, 0x0036, 0x002d, 0x006c, 0x0077, 0x005a, 0x0041,
0x00d8, 0x00c3, 0x00ee, 0x00f5, 0x00b4, 0x00af, 0x0082, 0x0099,
0x01b0, 0x01ab, 0x0186, 0x019d, 0x01dc, 0x01c7, 0x01ea, 0x01f1,
0x0168, 0x0173, 0x015e, 0x0145, 0x0104, 0x011f, 0x0132, 0x0129,
0x0360, 0x037b, 0x0356, 0x034d, 0x030c, 0x0317, 0x033a, 0x0321,
0x03b8, 0x03a3, 0x038e, 0x0395, 0x03d4, 0x03cf, 0x03e2, 0x03f9,
0x02d0, 0x02cb, 0x02e6, 0x02fd, 0x02bc, 0x02a7, 0x028a, 0x0291,
0x0208, 0x0213, 0x023e, 0x0225, 0x0264, 0x027f, 0x0252, 0x0249,
0x02db, 0x02c0, 0x02ed, 0x02f6, 0x02b7, 0x02ac, 0x0281, 0x029a,
0x0203, 0x0218, 0x0235, 0x022e, 0x026f, 0x0274, 0x0259, 0x0242,
0x036b, 0x0370, 0x035d, 0x0346, 0x0307, 0x031c, 0x0331, 0x032a,
0x03b3, 0x03a8, 0x0385, 0x039e, 0x03df, 0x03c4, 0x03e9, 0x03f2,
0x01bb, 0x01a0, 0x018d, 0x0196, 0x01d7, 0x01cc, 0x01e1, 0x01fa,
0x0163, 0x0178, 0x0155, 0x014e, 0x010f, 0x0114, 0x0139, 0x0122,
0x000b, 0x0010, 0x003d, 0x0026, 0x0067, 0x007c, 0x0051, 0x004a,
0x00d3, 0x00c8, 0x00e5, 0x00fe, 0x00bf, 0x00a4, 0x0089, 0x0092,
0x01ad, 0x01b6, 0x019b, 0x0180, 0x01c1, 0x01da, 0x01f7, 0x01ec,
0x0175, 0x016e, 0x0143, 0x0158, 0x0119, 0x0102, 0x012f, 0x0134,
0x001d, 0x0006, 0x002b, 0x0030, 0x0071, 0x006a, 0x0047, 0x005c,
0x00c5, 0x00de, 0x00f3, 0x00e8, 0x00a9, 0x00b2, 0x009f, 0x0084,
0x02cd, 0x02d6, 0x02fb, 0x02e0, 0x02a1, 0x02ba, 0x0297, 0x028c,
0x0215, 0x020e, 0x0223, 0x0238, 0x0279, 0x0262, 0x024f, 0x0254,
0x037d, 0x0366, 0x034b, 0x0350, 0x0311, 0x030a, 0x0327, 0x033c,
0x03a5, 0x03be, 0x0393, 0x0388, 0x03c9, 0x03d2, 0x03ff, 0x03e4,
0x0376, 0x036d, 0x0340, 0x035b, 0x031a, 0x0301, 0x032c, 0x0337,
0x03ae, 0x03b5, 0x0398, 0x0383, 0x03c2, 0x03d9, 0x03f4, 0x03ef,
0x02c6, 0x02dd, 0x02f0, 0x02eb, 0x02aa, 0x02b1, 0x029c, 0x0287,
0x021e, 0x0205, 0x0228, 0x0233, 0x0272, 0x0269, 0x0244, 0x025f,
0x0016, 0x000d, 0x0020, 0x003b, 0x007a, 0x0061, 0x004c, 0x0057,
0x00ce, 0x00d5, 0x00f8, 0x00e3, 0x00a2, 0x00b9, 0x0094, 0x008f,
0x01a6, 0x01bd, 0x0190, 0x018b, 0x01ca, 0x01d1, 0x01fc, 0x01e7,
0x017e, 0x0165, 0x0148, 0x0153, 0x0112, 0x0109, 0x0124, 0x013f,
};
/* This table is just the CRC of each possible byte which is used for
* Spectrum-4. It is computed, Msbit first, for the Bloom filter
* polynomial which is 0x2d (1 + x^2+ x^3 + x^5 and the implicit x^6).
*/
static const u8 mlxsw_sp4_acl_bf_crc6_tab[256] = {
0x00, 0x2d, 0x37, 0x1a, 0x03, 0x2e, 0x34, 0x19,
0x06, 0x2b, 0x31, 0x1c, 0x05, 0x28, 0x32, 0x1f,
0x0c, 0x21, 0x3b, 0x16, 0x0f, 0x22, 0x38, 0x15,
0x0a, 0x27, 0x3d, 0x10, 0x09, 0x24, 0x3e, 0x13,
0x18, 0x35, 0x2f, 0x02, 0x1b, 0x36, 0x2c, 0x01,
0x1e, 0x33, 0x29, 0x04, 0x1d, 0x30, 0x2a, 0x07,
0x14, 0x39, 0x23, 0x0e, 0x17, 0x3a, 0x20, 0x0d,
0x12, 0x3f, 0x25, 0x08, 0x11, 0x3c, 0x26, 0x0b,
0x30, 0x1d, 0x07, 0x2a, 0x33, 0x1e, 0x04, 0x29,
0x36, 0x1b, 0x01, 0x2c, 0x35, 0x18, 0x02, 0x2f,
0x3c, 0x11, 0x0b, 0x26, 0x3f, 0x12, 0x08, 0x25,
0x3a, 0x17, 0x0d, 0x20, 0x39, 0x14, 0x0e, 0x23,
0x28, 0x05, 0x1f, 0x32, 0x2b, 0x06, 0x1c, 0x31,
0x2e, 0x03, 0x19, 0x34, 0x2d, 0x00, 0x1a, 0x37,
0x24, 0x09, 0x13, 0x3e, 0x27, 0x0a, 0x10, 0x3d,
0x22, 0x0f, 0x15, 0x38, 0x21, 0x0c, 0x16, 0x3b,
0x0d, 0x20, 0x3a, 0x17, 0x0e, 0x23, 0x39, 0x14,
0x0b, 0x26, 0x3c, 0x11, 0x08, 0x25, 0x3f, 0x12,
0x01, 0x2c, 0x36, 0x1b, 0x02, 0x2f, 0x35, 0x18,
0x07, 0x2a, 0x30, 0x1d, 0x04, 0x29, 0x33, 0x1e,
0x15, 0x38, 0x22, 0x0f, 0x16, 0x3b, 0x21, 0x0c,
0x13, 0x3e, 0x24, 0x09, 0x10, 0x3d, 0x27, 0x0a,
0x19, 0x34, 0x2e, 0x03, 0x1a, 0x37, 0x2d, 0x00,
0x1f, 0x32, 0x28, 0x05, 0x1c, 0x31, 0x2b, 0x06,
0x3d, 0x10, 0x0a, 0x27, 0x3e, 0x13, 0x09, 0x24,
0x3b, 0x16, 0x0c, 0x21, 0x38, 0x15, 0x0f, 0x22,
0x31, 0x1c, 0x06, 0x2b, 0x32, 0x1f, 0x05, 0x28,
0x37, 0x1a, 0x00, 0x2d, 0x34, 0x19, 0x03, 0x2e,
0x25, 0x08, 0x12, 0x3f, 0x26, 0x0b, 0x11, 0x3c,
0x23, 0x0e, 0x14, 0x39, 0x20, 0x0d, 0x17, 0x3a,
0x29, 0x04, 0x1e, 0x33, 0x2a, 0x07, 0x1d, 0x30,
0x2f, 0x02, 0x18, 0x35, 0x2c, 0x01, 0x1b, 0x36,
};
/* Each chunk contains 4 key blocks. Chunk 2 uses key blocks 11-8,
* and we need to populate it with 4 key blocks copied from the entry encoded
* key. The original keys layout is same for Spectrum-{2,3,4}.
* Since the encoded key contains a 2 bytes padding, key block 11 starts at
* offset 2. block 7 that is used in chunk 1 starts at offset 20 as 4 key blocks
* take 18 bytes. See 'MLXSW_SP2_AFK_BLOCK_LAYOUT' for more details.
* This array defines key offsets for easy access when copying key blocks from
* entry key to Bloom filter chunk.
*/
static const u8 chunk_key_offsets[MLXSW_BLOOM_KEY_CHUNKS] = {2, 20, 38};
static u16 mlxsw_sp2_acl_bf_crc16_byte(u16 crc, u8 c)
{
return (crc << 8) ^ mlxsw_sp_acl_bf_crc_tab[(crc >> 8) ^ c];
return (crc << 8) ^ mlxsw_sp2_acl_bf_crc16_tab[(crc >> 8) ^ c];
}
static u16 mlxsw_sp_acl_bf_crc(const u8 *buffer, size_t len)
static u16 mlxsw_sp2_acl_bf_crc(const u8 *buffer, size_t len)
{
u16 crc = 0;
while (len--)
crc = mlxsw_sp_acl_bf_crc_byte(crc, *buffer++);
crc = mlxsw_sp2_acl_bf_crc16_byte(crc, *buffer++);
return crc;
}
static void
mlxsw_sp_acl_bf_key_encode(struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry,
char *output, u8 *len)
__mlxsw_sp_acl_bf_key_encode(struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry,
char *output, u8 *len, u8 max_chunks, u8 pad_bytes,
u8 key_offset, u8 chunk_key_len, u8 chunk_len)
{
struct mlxsw_afk_key_info *key_info = aregion->region->key_info;
u8 chunk_index, chunk_count, block_count;
......@@ -129,37 +243,168 @@ mlxsw_sp_acl_bf_key_encode(struct mlxsw_sp_acl_atcam_region *aregion,
chunk_count = 1 + ((block_count - 1) >> 2);
erp_region_id = cpu_to_be16(aentry->ht_key.erp_id |
(aregion->region->id << 4));
for (chunk_index = MLXSW_BLOOM_KEY_CHUNKS - chunk_count;
chunk_index < MLXSW_BLOOM_KEY_CHUNKS; chunk_index++) {
memset(chunk, 0, MLXSW_BLOOM_CHUNK_PAD_BYTES);
memcpy(chunk + MLXSW_BLOOM_CHUNK_PAD_BYTES, &erp_region_id,
for (chunk_index = max_chunks - chunk_count; chunk_index < max_chunks;
chunk_index++) {
memset(chunk, 0, pad_bytes);
memcpy(chunk + pad_bytes, &erp_region_id,
sizeof(erp_region_id));
memcpy(chunk + MLXSW_BLOOM_CHUNK_KEY_OFFSET,
memcpy(chunk + key_offset,
&aentry->enc_key[chunk_key_offsets[chunk_index]],
MLXSW_BLOOM_CHUNK_KEY_BYTES);
chunk += MLXSW_BLOOM_KEY_CHUNK_BYTES;
chunk_key_len);
chunk += chunk_len;
}
*len = chunk_count * MLXSW_BLOOM_KEY_CHUNK_BYTES;
*len = chunk_count * chunk_len;
}
static void
mlxsw_sp2_acl_bf_key_encode(struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry,
char *output, u8 *len)
{
__mlxsw_sp_acl_bf_key_encode(aregion, aentry, output, len,
MLXSW_BLOOM_KEY_CHUNKS,
MLXSW_SP2_BLOOM_CHUNK_PAD_BYTES,
MLXSW_SP2_BLOOM_CHUNK_KEY_OFFSET,
MLXSW_SP2_BLOOM_CHUNK_KEY_BYTES,
MLXSW_SP2_BLOOM_KEY_CHUNK_BYTES);
}
static unsigned int
mlxsw_sp_acl_bf_rule_count_index_get(struct mlxsw_sp_acl_bf *bf,
unsigned int erp_bank,
unsigned int bf_index)
mlxsw_sp2_acl_bf_index_get(struct mlxsw_sp_acl_bf *bf,
struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry)
{
return erp_bank * bf->bank_size + bf_index;
char bf_key[MLXSW_SP2_BLOOM_KEY_LEN];
u8 bf_size;
mlxsw_sp2_acl_bf_key_encode(aregion, aentry, bf_key, &bf_size);
return mlxsw_sp2_acl_bf_crc(bf_key, bf_size);
}
static u16 mlxsw_sp4_acl_bf_crc10_byte(u16 crc, u8 c)
{
u8 index = ((crc >> 2) ^ c) & 0xff;
return ((crc << 8) ^ mlxsw_sp4_acl_bf_crc10_tab[index]) & 0x3ff;
}
static u16 mlxsw_sp4_acl_bf_crc6_byte(u16 crc, u8 c)
{
u8 index = (crc ^ c) & 0xff;
return ((crc << 6) ^ (mlxsw_sp4_acl_bf_crc6_tab[index] << 2)) & 0xfc;
}
static u16 mlxsw_sp4_acl_bf_crc(const u8 *buffer, size_t len)
{
u16 crc_row = 0, crc_col = 0;
while (len--) {
crc_row = mlxsw_sp4_acl_bf_crc10_byte(crc_row, *buffer);
crc_col = mlxsw_sp4_acl_bf_crc6_byte(crc_col, *buffer);
buffer++;
}
crc_col >>= 2;
/* 6 bit column are MSB, 10 bit row are LSB */
return (crc_col << 10) | crc_row;
}
static void right_shift_array(char *arr, u8 len, u8 shift_bits)
{
u8 byte_mask = 0xff >> shift_bits;
int i;
if (WARN_ON(!shift_bits || shift_bits >= 8))
return;
for (i = len - 1; i >= 0; i--) {
/* The first iteration looks like out-of-bounds access,
* but actually references a buffer that the array is shifted
* into. This move is legal as we never send the last chunk to
* this function.
*/
arr[i + 1] &= byte_mask;
arr[i + 1] |= arr[i] << (8 - shift_bits);
arr[i] = arr[i] >> shift_bits;
}
}
static void mlxsw_sp4_bf_key_shift_chunks(u8 chunk_count, char *output)
{
/* The chunks are suppoosed to be continuous, with no padding.
* Since region ID and eRP ID use 14 bits, and not fully 2 bytes,
* and in Spectrum-4 there is no padding, it is necessary to shift some
* chunks 2 bits right.
*/
switch (chunk_count) {
case 2:
/* The chunks are copied as follow:
* +-------------+-----------------+
* | Chunk 0 | Chunk 1 |
* | IDs | keys |(**) IDs | keys |
* +-------------+-----------------+
* In (**), there are two unused bits, therefore, chunk 0 needs
* to be shifted two bits right.
*/
right_shift_array(output, MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES, 2);
break;
case 3:
/* The chunks are copied as follow:
* +-------------+-----------------+-----------------+
* | Chunk 0 | Chunk 1 | Chunk 2 |
* | IDs | keys |(**) IDs | keys |(**) IDs | keys |
* +-------------+-----------------+-----------------+
* In (**), there are two unused bits, therefore, chunk 1 needs
* to be shifted two bits right and chunk 0 needs to be shifted
* four bits right.
*/
right_shift_array(output + MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES,
MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES, 2);
right_shift_array(output, MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES, 4);
break;
default:
WARN_ON(chunk_count > MLXSW_BLOOM_KEY_CHUNKS);
}
}
static void
mlxsw_sp4_acl_bf_key_encode(struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry,
char *output, u8 *len)
{
struct mlxsw_afk_key_info *key_info = aregion->region->key_info;
u8 block_count = mlxsw_afk_key_info_blocks_count_get(key_info);
u8 chunk_count = 1 + ((block_count - 1) >> 2);
__mlxsw_sp_acl_bf_key_encode(aregion, aentry, output, len,
MLXSW_BLOOM_KEY_CHUNKS,
MLXSW_SP4_BLOOM_CHUNK_PAD_BYTES,
MLXSW_SP4_BLOOM_CHUNK_KEY_OFFSET,
MLXSW_SP4_BLOOM_CHUNK_KEY_BYTES,
MLXSW_SP4_BLOOM_KEY_CHUNK_BYTES);
mlxsw_sp4_bf_key_shift_chunks(chunk_count, output);
}
static unsigned int
mlxsw_sp_acl_bf_index_get(struct mlxsw_sp_acl_bf *bf,
struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry)
mlxsw_sp4_acl_bf_index_get(struct mlxsw_sp_acl_bf *bf,
struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry)
{
char bf_key[MLXSW_BLOOM_KEY_LEN];
char bf_key[MLXSW_SP4_BLOOM_KEY_LEN] = {};
u8 bf_size;
mlxsw_sp_acl_bf_key_encode(aregion, aentry, bf_key, &bf_size);
return mlxsw_sp_acl_bf_crc(bf_key, bf_size);
mlxsw_sp4_acl_bf_key_encode(aregion, aentry, bf_key, &bf_size);
return mlxsw_sp4_acl_bf_crc(bf_key, bf_size);
}
static unsigned int
mlxsw_sp_acl_bf_rule_count_index_get(struct mlxsw_sp_acl_bf *bf,
unsigned int erp_bank,
unsigned int bf_index)
{
return erp_bank * bf->bank_size + bf_index;
}
int
......@@ -176,7 +421,7 @@ mlxsw_sp_acl_bf_entry_add(struct mlxsw_sp *mlxsw_sp,
mutex_lock(&bf->lock);
bf_index = mlxsw_sp_acl_bf_index_get(bf, aregion, aentry);
bf_index = mlxsw_sp->acl_bf_ops->index_get(bf, aregion, aentry);
rule_index = mlxsw_sp_acl_bf_rule_count_index_get(bf, erp_bank,
bf_index);
......@@ -219,7 +464,7 @@ mlxsw_sp_acl_bf_entry_del(struct mlxsw_sp *mlxsw_sp,
mutex_lock(&bf->lock);
bf_index = mlxsw_sp_acl_bf_index_get(bf, aregion, aentry);
bf_index = mlxsw_sp->acl_bf_ops->index_get(bf, aregion, aentry);
rule_index = mlxsw_sp_acl_bf_rule_count_index_get(bf, erp_bank,
bf_index);
......@@ -267,3 +512,11 @@ void mlxsw_sp_acl_bf_fini(struct mlxsw_sp_acl_bf *bf)
mutex_destroy(&bf->lock);
kfree(bf);
}
const struct mlxsw_sp_acl_bf_ops mlxsw_sp2_acl_bf_ops = {
.index_get = mlxsw_sp2_acl_bf_index_get,
};
const struct mlxsw_sp_acl_bf_ops mlxsw_sp4_acl_bf_ops = {
.index_get = mlxsw_sp4_acl_bf_index_get,
};
......@@ -168,8 +168,8 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_2[] = {
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_4[] = {
MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_0_7, 0x04, 24, 8),
MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_8_10, 0x00, 0, 3),
MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_LSB, 0x04, 24, 8),
MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_MSB, 0x00, 0, 3),
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv6_0[] = {
......@@ -311,3 +311,45 @@ const struct mlxsw_afk_ops mlxsw_sp2_afk_ops = {
.encode_block = mlxsw_sp2_afk_encode_block,
.clear_block = mlxsw_sp2_afk_clear_block,
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_mac_5b[] = {
MLXSW_AFK_ELEMENT_INST_U32(VID, 0x04, 18, 12),
MLXSW_AFK_ELEMENT_INST_EXT_U32(SRC_SYS_PORT, 0x04, 0, 9, -1, true), /* RX_ACL_SYSTEM_PORT */
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_4b[] = {
MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_LSB, 0x04, 13, 8),
MLXSW_AFK_ELEMENT_INST_EXT_U32(VIRT_ROUTER_MSB, 0x04, 21, 4, 0, true),
};
static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv6_2b[] = {
MLXSW_AFK_ELEMENT_INST_BUF(DST_IP_96_127, 0x04, 4),
};
static const struct mlxsw_afk_block mlxsw_sp4_afk_blocks[] = {
MLXSW_AFK_BLOCK(0x10, mlxsw_sp_afk_element_info_mac_0),
MLXSW_AFK_BLOCK(0x11, mlxsw_sp_afk_element_info_mac_1),
MLXSW_AFK_BLOCK(0x12, mlxsw_sp_afk_element_info_mac_2),
MLXSW_AFK_BLOCK(0x13, mlxsw_sp_afk_element_info_mac_3),
MLXSW_AFK_BLOCK(0x14, mlxsw_sp_afk_element_info_mac_4),
MLXSW_AFK_BLOCK(0x1A, mlxsw_sp_afk_element_info_mac_5b),
MLXSW_AFK_BLOCK(0x38, mlxsw_sp_afk_element_info_ipv4_0),
MLXSW_AFK_BLOCK(0x39, mlxsw_sp_afk_element_info_ipv4_1),
MLXSW_AFK_BLOCK(0x3A, mlxsw_sp_afk_element_info_ipv4_2),
MLXSW_AFK_BLOCK(0x35, mlxsw_sp_afk_element_info_ipv4_4b),
MLXSW_AFK_BLOCK(0x40, mlxsw_sp_afk_element_info_ipv6_0),
MLXSW_AFK_BLOCK(0x41, mlxsw_sp_afk_element_info_ipv6_1),
MLXSW_AFK_BLOCK(0x47, mlxsw_sp_afk_element_info_ipv6_2b),
MLXSW_AFK_BLOCK(0x43, mlxsw_sp_afk_element_info_ipv6_3),
MLXSW_AFK_BLOCK(0x44, mlxsw_sp_afk_element_info_ipv6_4),
MLXSW_AFK_BLOCK(0x45, mlxsw_sp_afk_element_info_ipv6_5),
MLXSW_AFK_BLOCK(0x90, mlxsw_sp_afk_element_info_l4_0),
MLXSW_AFK_BLOCK(0x92, mlxsw_sp_afk_element_info_l4_2),
};
const struct mlxsw_afk_ops mlxsw_sp4_afk_ops = {
.blocks = mlxsw_sp4_afk_blocks,
.blocks_count = ARRAY_SIZE(mlxsw_sp4_afk_blocks),
.encode_block = mlxsw_sp2_afk_encode_block,
.clear_block = mlxsw_sp2_afk_clear_block,
};
......@@ -287,6 +287,12 @@ void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_acl_bf;
struct mlxsw_sp_acl_bf_ops {
unsigned int (*index_get)(struct mlxsw_sp_acl_bf *bf,
struct mlxsw_sp_acl_atcam_region *aregion,
struct mlxsw_sp_acl_atcam_entry *aentry);
};
int
mlxsw_sp_acl_bf_entry_add(struct mlxsw_sp *mlxsw_sp,
struct mlxsw_sp_acl_bf *bf,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment