Commit 8a015bae authored by Yevgeny Kliteynik's avatar Yevgeny Kliteynik Committed by Saeed Mahameed

net/mlx5: DR, Improve rule tracking memory consumption

To track each STE of the rule a rule member was allocated, each
member would point to one STE. This means that we would allocate
40B (rule member) * number of STEs per rule.

To reduce this per rule allocation we use the STE tree pointers
for next_htbl and pointing STE to navigate the tree, this allows
us to keep only the pointer to the last STE of rule (always unique).
From the last rule STE we are able to traverse and rebuild all of
the STEs that construct the rule.

In our testing with 8M rules, each consisting of 7 STES, we were able
to reduce 1.6GB of memory.
Signed-off-by: default avatarAlex Vesker <valex@nvidia.com>
Signed-off-by: default avatarErez Shitrit <erezsh@nvidia.com>
Signed-off-by: default avatarYevgeny Kliteynik <kliteyn@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 32c8e3b2
...@@ -81,6 +81,7 @@ dr_rule_create_collision_entry(struct mlx5dr_matcher *matcher, ...@@ -81,6 +81,7 @@ dr_rule_create_collision_entry(struct mlx5dr_matcher *matcher,
} }
ste->ste_chain_location = orig_ste->ste_chain_location; ste->ste_chain_location = orig_ste->ste_chain_location;
ste->htbl->pointing_ste = orig_ste->htbl->pointing_ste;
/* In collision entry, all members share the same miss_list_head */ /* In collision entry, all members share the same miss_list_head */
ste->htbl->miss_list = mlx5dr_ste_get_miss_list(orig_ste); ste->htbl->miss_list = mlx5dr_ste_get_miss_list(orig_ste);
...@@ -185,6 +186,9 @@ dr_rule_rehash_handle_collision(struct mlx5dr_matcher *matcher, ...@@ -185,6 +186,9 @@ dr_rule_rehash_handle_collision(struct mlx5dr_matcher *matcher,
if (!new_ste) if (!new_ste)
return NULL; return NULL;
/* Update collision pointing STE */
new_ste->htbl->pointing_ste = col_ste->htbl->pointing_ste;
/* In collision entry, all members share the same miss_list_head */ /* In collision entry, all members share the same miss_list_head */
new_ste->htbl->miss_list = mlx5dr_ste_get_miss_list(col_ste); new_ste->htbl->miss_list = mlx5dr_ste_get_miss_list(col_ste);
...@@ -212,7 +216,7 @@ static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher, ...@@ -212,7 +216,7 @@ static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher,
new_ste->next_htbl = cur_ste->next_htbl; new_ste->next_htbl = cur_ste->next_htbl;
new_ste->ste_chain_location = cur_ste->ste_chain_location; new_ste->ste_chain_location = cur_ste->ste_chain_location;
if (!mlx5dr_ste_is_last_in_rule(nic_matcher, new_ste->ste_chain_location)) if (new_ste->next_htbl)
new_ste->next_htbl->pointing_ste = new_ste; new_ste->next_htbl->pointing_ste = new_ste;
/* We need to copy the refcount since this ste /* We need to copy the refcount since this ste
...@@ -220,10 +224,8 @@ static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher, ...@@ -220,10 +224,8 @@ static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher,
*/ */
new_ste->refcount = cur_ste->refcount; new_ste->refcount = cur_ste->refcount;
/* Link old STEs rule_mem list to the new ste */ /* Link old STEs rule to the new ste */
mlx5dr_rule_update_rule_member(cur_ste, new_ste); mlx5dr_rule_set_last_member(cur_ste->rule_rx_tx, new_ste, false);
INIT_LIST_HEAD(&new_ste->rule_list);
list_splice_tail_init(&cur_ste->rule_list, &new_ste->rule_list);
} }
static struct mlx5dr_ste * static struct mlx5dr_ste *
...@@ -581,34 +583,66 @@ static int dr_rule_add_action_members(struct mlx5dr_rule *rule, ...@@ -581,34 +583,66 @@ static int dr_rule_add_action_members(struct mlx5dr_rule *rule,
return -ENOMEM; return -ENOMEM;
} }
/* While the pointer of ste is no longer valid, like while moving ste to be void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
* the first in the miss_list, and to be in the origin table, struct mlx5dr_ste *ste,
* all rule-members that are attached to this ste should update their ste member bool force)
* to the new pointer {
*/ /* Update rule member is usually done for the last STE or during rule
void mlx5dr_rule_update_rule_member(struct mlx5dr_ste *ste, * creation to recover from mid-creation failure (for this peruse the
struct mlx5dr_ste *new_ste) * force flag is used)
*/
if (ste->next_htbl && !force)
return;
/* Update is required since each rule keeps track of its last STE */
ste->rule_rx_tx = nic_rule;
nic_rule->last_rule_ste = ste;
}
static struct mlx5dr_ste *dr_rule_get_pointed_ste(struct mlx5dr_ste *curr_ste)
{
struct mlx5dr_ste *first_ste;
first_ste = list_first_entry(mlx5dr_ste_get_miss_list(curr_ste),
struct mlx5dr_ste, miss_list_node);
return first_ste->htbl->pointing_ste;
}
int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
struct mlx5dr_ste *curr_ste,
int *num_of_stes)
{ {
struct mlx5dr_rule_member *rule_mem; bool first = false;
*num_of_stes = 0;
if (!curr_ste)
return -ENOENT;
/* Iterate from last to first */
while (!first) {
first = curr_ste->ste_chain_location == 1;
ste_arr[*num_of_stes] = curr_ste;
*num_of_stes += 1;
curr_ste = dr_rule_get_pointed_ste(curr_ste);
}
list_for_each_entry(rule_mem, &ste->rule_list, use_ste_list) return 0;
rule_mem->ste = new_ste;
} }
static void dr_rule_clean_rule_members(struct mlx5dr_rule *rule, static void dr_rule_clean_rule_members(struct mlx5dr_rule *rule,
struct mlx5dr_rule_rx_tx *nic_rule) struct mlx5dr_rule_rx_tx *nic_rule)
{ {
struct mlx5dr_rule_member *rule_mem; struct mlx5dr_ste *ste_arr[DR_RULE_MAX_STES + DR_ACTION_MAX_STES];
struct mlx5dr_rule_member *tmp_mem; struct mlx5dr_ste *curr_ste = nic_rule->last_rule_ste;
int i;
if (list_empty(&nic_rule->rule_members_list)) if (mlx5dr_rule_get_reverse_rule_members(ste_arr, curr_ste, &i))
return; return;
list_for_each_entry_safe(rule_mem, tmp_mem, &nic_rule->rule_members_list, list) {
list_del(&rule_mem->list); while (i--)
list_del(&rule_mem->use_ste_list); mlx5dr_ste_put(ste_arr[i], rule->matcher, nic_rule->nic_matcher);
mlx5dr_ste_put(rule_mem->ste, rule->matcher, nic_rule->nic_matcher);
kvfree(rule_mem);
}
} }
static u16 dr_get_bits_per_mask(u16 byte_mask) static u16 dr_get_bits_per_mask(u16 byte_mask)
...@@ -647,26 +681,6 @@ static bool dr_rule_need_enlarge_hash(struct mlx5dr_ste_htbl *htbl, ...@@ -647,26 +681,6 @@ static bool dr_rule_need_enlarge_hash(struct mlx5dr_ste_htbl *htbl,
return false; return false;
} }
static int dr_rule_add_member(struct mlx5dr_rule_rx_tx *nic_rule,
struct mlx5dr_ste *ste)
{
struct mlx5dr_rule_member *rule_mem;
rule_mem = kvzalloc(sizeof(*rule_mem), GFP_KERNEL);
if (!rule_mem)
return -ENOMEM;
INIT_LIST_HEAD(&rule_mem->list);
INIT_LIST_HEAD(&rule_mem->use_ste_list);
rule_mem->ste = ste;
list_add_tail(&rule_mem->list, &nic_rule->rule_members_list);
list_add_tail(&rule_mem->use_ste_list, &ste->rule_list);
return 0;
}
static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule, static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule,
struct mlx5dr_rule_rx_tx *nic_rule, struct mlx5dr_rule_rx_tx *nic_rule,
struct list_head *send_ste_list, struct list_head *send_ste_list,
...@@ -681,15 +695,13 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule, ...@@ -681,15 +695,13 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule,
struct mlx5dr_domain *dmn = matcher->tbl->dmn; struct mlx5dr_domain *dmn = matcher->tbl->dmn;
u8 *curr_hw_ste, *prev_hw_ste; u8 *curr_hw_ste, *prev_hw_ste;
struct mlx5dr_ste *action_ste; struct mlx5dr_ste *action_ste;
int i, k, ret; int i, k;
/* Two cases: /* Two cases:
* 1. num_of_builders is equal to new_hw_ste_arr_sz, the action in the ste * 1. num_of_builders is equal to new_hw_ste_arr_sz, the action in the ste
* 2. num_of_builders is less then new_hw_ste_arr_sz, new ste was added * 2. num_of_builders is less then new_hw_ste_arr_sz, new ste was added
* to support the action. * to support the action.
*/ */
if (num_of_builders == new_hw_ste_arr_sz)
return 0;
for (i = num_of_builders, k = 0; i < new_hw_ste_arr_sz; i++, k++) { for (i = num_of_builders, k = 0; i < new_hw_ste_arr_sz; i++, k++) {
curr_hw_ste = hw_ste_arr + i * DR_STE_SIZE; curr_hw_ste = hw_ste_arr + i * DR_STE_SIZE;
...@@ -702,6 +714,10 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule, ...@@ -702,6 +714,10 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule,
mlx5dr_ste_get(action_ste); mlx5dr_ste_get(action_ste);
action_ste->htbl->pointing_ste = last_ste;
last_ste->next_htbl = action_ste->htbl;
last_ste = action_ste;
/* While free ste we go over the miss list, so add this ste to the list */ /* While free ste we go over the miss list, so add this ste to the list */
list_add_tail(&action_ste->miss_list_node, list_add_tail(&action_ste->miss_list_node,
mlx5dr_ste_get_miss_list(action_ste)); mlx5dr_ste_get_miss_list(action_ste));
...@@ -715,21 +731,19 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule, ...@@ -715,21 +731,19 @@ static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule,
mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx, mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx,
prev_hw_ste, prev_hw_ste,
action_ste->htbl); action_ste->htbl);
ret = dr_rule_add_member(nic_rule, action_ste);
if (ret) { mlx5dr_rule_set_last_member(nic_rule, action_ste, true);
mlx5dr_dbg(dmn, "Failed adding rule member\n");
goto free_ste_info;
}
mlx5dr_send_fill_and_append_ste_send_info(action_ste, DR_STE_SIZE, 0, mlx5dr_send_fill_and_append_ste_send_info(action_ste, DR_STE_SIZE, 0,
curr_hw_ste, curr_hw_ste,
ste_info_arr[k], ste_info_arr[k],
send_ste_list, false); send_ste_list, false);
} }
last_ste->next_htbl = NULL;
return 0; return 0;
free_ste_info:
kfree(ste_info_arr[k]);
err_exit: err_exit:
mlx5dr_ste_put(action_ste, matcher, nic_matcher); mlx5dr_ste_put(action_ste, matcher, nic_matcher);
return -ENOMEM; return -ENOMEM;
...@@ -1067,8 +1081,6 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, ...@@ -1067,8 +1081,6 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule,
nic_matcher = nic_rule->nic_matcher; nic_matcher = nic_rule->nic_matcher;
nic_dmn = nic_matcher->nic_tbl->nic_dmn; nic_dmn = nic_matcher->nic_tbl->nic_dmn;
INIT_LIST_HEAD(&nic_rule->rule_members_list);
if (dr_rule_skip(dmn->type, nic_dmn->type, &matcher->mask, param, if (dr_rule_skip(dmn->type, nic_dmn->type, &matcher->mask, param,
rule->flow_source)) rule->flow_source))
return 0; return 0;
...@@ -1123,14 +1135,8 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, ...@@ -1123,14 +1135,8 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule,
cur_htbl = ste->next_htbl; cur_htbl = ste->next_htbl;
/* Keep all STEs in the rule struct */
ret = dr_rule_add_member(nic_rule, ste);
if (ret) {
mlx5dr_dbg(dmn, "Failed adding rule member index %d\n", i);
goto free_ste;
}
mlx5dr_ste_get(ste); mlx5dr_ste_get(ste);
mlx5dr_rule_set_last_member(nic_rule, ste, true);
} }
/* Connect actions */ /* Connect actions */
...@@ -1155,8 +1161,6 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, ...@@ -1155,8 +1161,6 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule,
return 0; return 0;
free_ste:
mlx5dr_ste_put(ste, matcher, nic_matcher);
free_rule: free_rule:
dr_rule_clean_rule_members(rule, nic_rule); dr_rule_clean_rule_members(rule, nic_rule);
/* Clean all ste_info's */ /* Clean all ste_info's */
......
...@@ -172,9 +172,6 @@ static void dr_ste_replace(struct mlx5dr_ste *dst, struct mlx5dr_ste *src) ...@@ -172,9 +172,6 @@ static void dr_ste_replace(struct mlx5dr_ste *dst, struct mlx5dr_ste *src)
dst->next_htbl->pointing_ste = dst; dst->next_htbl->pointing_ste = dst;
dst->refcount = src->refcount; dst->refcount = src->refcount;
INIT_LIST_HEAD(&dst->rule_list);
list_splice_tail_init(&src->rule_list, &dst->rule_list);
} }
/* Free ste which is the head and the only one in miss_list */ /* Free ste which is the head and the only one in miss_list */
...@@ -233,12 +230,12 @@ dr_ste_replace_head_ste(struct mlx5dr_matcher_rx_tx *nic_matcher, ...@@ -233,12 +230,12 @@ dr_ste_replace_head_ste(struct mlx5dr_matcher_rx_tx *nic_matcher,
/* Remove from the miss_list the next_ste before copy */ /* Remove from the miss_list the next_ste before copy */
list_del_init(&next_ste->miss_list_node); list_del_init(&next_ste->miss_list_node);
/* All rule-members that use next_ste should know about that */
mlx5dr_rule_update_rule_member(next_ste, ste);
/* Move data from next into ste */ /* Move data from next into ste */
dr_ste_replace(ste, next_ste); dr_ste_replace(ste, next_ste);
/* Update the rule on STE change */
mlx5dr_rule_set_last_member(next_ste->rule_rx_tx, ste, false);
/* Copy all 64 hw_ste bytes */ /* Copy all 64 hw_ste bytes */
memcpy(hw_ste, ste->hw_ste, DR_STE_SIZE_REDUCED); memcpy(hw_ste, ste->hw_ste, DR_STE_SIZE_REDUCED);
sb_idx = ste->ste_chain_location - 1; sb_idx = ste->ste_chain_location - 1;
...@@ -499,7 +496,6 @@ struct mlx5dr_ste_htbl *mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool, ...@@ -499,7 +496,6 @@ struct mlx5dr_ste_htbl *mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
ste->refcount = 0; ste->refcount = 0;
INIT_LIST_HEAD(&ste->miss_list_node); INIT_LIST_HEAD(&ste->miss_list_node);
INIT_LIST_HEAD(&htbl->miss_list[i]); INIT_LIST_HEAD(&htbl->miss_list[i]);
INIT_LIST_HEAD(&ste->rule_list);
} }
htbl->chunk_size = chunk_size; htbl->chunk_size = chunk_size;
......
...@@ -140,6 +140,7 @@ struct mlx5dr_icm_buddy_mem; ...@@ -140,6 +140,7 @@ struct mlx5dr_icm_buddy_mem;
struct mlx5dr_ste_htbl; struct mlx5dr_ste_htbl;
struct mlx5dr_match_param; struct mlx5dr_match_param;
struct mlx5dr_cmd_caps; struct mlx5dr_cmd_caps;
struct mlx5dr_rule_rx_tx;
struct mlx5dr_matcher_rx_tx; struct mlx5dr_matcher_rx_tx;
struct mlx5dr_ste_ctx; struct mlx5dr_ste_ctx;
...@@ -151,14 +152,14 @@ struct mlx5dr_ste { ...@@ -151,14 +152,14 @@ struct mlx5dr_ste {
/* attached to the miss_list head at each htbl entry */ /* attached to the miss_list head at each htbl entry */
struct list_head miss_list_node; struct list_head miss_list_node;
/* each rule member that uses this ste attached here */
struct list_head rule_list;
/* this ste is member of htbl */ /* this ste is member of htbl */
struct mlx5dr_ste_htbl *htbl; struct mlx5dr_ste_htbl *htbl;
struct mlx5dr_ste_htbl *next_htbl; struct mlx5dr_ste_htbl *next_htbl;
/* The rule this STE belongs to */
struct mlx5dr_rule_rx_tx *rule_rx_tx;
/* this ste is part of a rule, located in ste's chain */ /* this ste is part of a rule, located in ste's chain */
u8 ste_chain_location; u8 ste_chain_location;
}; };
...@@ -888,14 +889,6 @@ struct mlx5dr_matcher { ...@@ -888,14 +889,6 @@ struct mlx5dr_matcher {
struct mlx5dv_flow_matcher *dv_matcher; struct mlx5dv_flow_matcher *dv_matcher;
}; };
struct mlx5dr_rule_member {
struct mlx5dr_ste *ste;
/* attached to mlx5dr_rule via this */
struct list_head list;
/* attached to mlx5dr_ste via this */
struct list_head use_ste_list;
};
struct mlx5dr_ste_action_modify_field { struct mlx5dr_ste_action_modify_field {
u16 hw_field; u16 hw_field;
u8 start; u8 start;
...@@ -996,8 +989,8 @@ struct mlx5dr_htbl_connect_info { ...@@ -996,8 +989,8 @@ struct mlx5dr_htbl_connect_info {
}; };
struct mlx5dr_rule_rx_tx { struct mlx5dr_rule_rx_tx {
struct list_head rule_members_list;
struct mlx5dr_matcher_rx_tx *nic_matcher; struct mlx5dr_matcher_rx_tx *nic_matcher;
struct mlx5dr_ste *last_rule_ste;
}; };
struct mlx5dr_rule { struct mlx5dr_rule {
...@@ -1008,8 +1001,12 @@ struct mlx5dr_rule { ...@@ -1008,8 +1001,12 @@ struct mlx5dr_rule {
u32 flow_source; u32 flow_source;
}; };
void mlx5dr_rule_update_rule_member(struct mlx5dr_ste *new_ste, void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
struct mlx5dr_ste *ste); struct mlx5dr_ste *ste,
bool force);
int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
struct mlx5dr_ste *curr_ste,
int *num_of_stes);
struct mlx5dr_icm_chunk { struct mlx5dr_icm_chunk {
struct mlx5dr_icm_buddy_mem *buddy_mem; struct mlx5dr_icm_buddy_mem *buddy_mem;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment