Commit 5828cd9a authored by Andy Zhou's avatar Andy Zhou Committed by Jesse Gross

openvswitch: optimize flow compare and mask functions

Make sure the sw_flow_key structure and valid mask boundaries are always
machine word aligned. Optimize the flow compare and mask operations
using machine word size operations. This patch improves throughput on
average by 15% when CPU is the bottleneck of forwarding packets.

This patch is inspired by ideas and code from a patch submitted by Peter
Klausler titled "replace memcmp() with specialized comparator".
However, The original patch only optimizes for architectures
support unaligned machine word access. This patch optimizes for all
architectures.
Signed-off-by: default avatarAndy Zhou <azhou@nicira.com>
Signed-off-by: default avatarJesse Gross <jesse@nicira.com>
parent 02237373
...@@ -54,8 +54,8 @@ static void update_range__(struct sw_flow_match *match, ...@@ -54,8 +54,8 @@ static void update_range__(struct sw_flow_match *match,
size_t offset, size_t size, bool is_mask) size_t offset, size_t size, bool is_mask)
{ {
struct sw_flow_key_range *range = NULL; struct sw_flow_key_range *range = NULL;
size_t start = offset; size_t start = rounddown(offset, sizeof(long));
size_t end = offset + size; size_t end = roundup(offset + size, sizeof(long));
if (!is_mask) if (!is_mask)
range = &match->range; range = &match->range;
...@@ -102,6 +102,11 @@ static void update_range__(struct sw_flow_match *match, ...@@ -102,6 +102,11 @@ static void update_range__(struct sw_flow_match *match,
} \ } \
} while (0) } while (0)
static u16 range_n_bytes(const struct sw_flow_key_range *range)
{
return range->end - range->start;
}
void ovs_match_init(struct sw_flow_match *match, void ovs_match_init(struct sw_flow_match *match,
struct sw_flow_key *key, struct sw_flow_key *key,
struct sw_flow_mask *mask) struct sw_flow_mask *mask)
...@@ -370,16 +375,17 @@ static bool icmp6hdr_ok(struct sk_buff *skb) ...@@ -370,16 +375,17 @@ static bool icmp6hdr_ok(struct sk_buff *skb)
void ovs_flow_key_mask(struct sw_flow_key *dst, const struct sw_flow_key *src, void ovs_flow_key_mask(struct sw_flow_key *dst, const struct sw_flow_key *src,
const struct sw_flow_mask *mask) const struct sw_flow_mask *mask)
{ {
u8 *m = (u8 *)&mask->key + mask->range.start; const long *m = (long *)((u8 *)&mask->key + mask->range.start);
u8 *s = (u8 *)src + mask->range.start; const long *s = (long *)((u8 *)src + mask->range.start);
u8 *d = (u8 *)dst + mask->range.start; long *d = (long *)((u8 *)dst + mask->range.start);
int i; int i;
memset(dst, 0, sizeof(*dst)); /* The memory outside of the 'mask->range' are not set since
for (i = 0; i < ovs_sw_flow_mask_size_roundup(mask); i++) { * further operations on 'dst' only uses contents within
*d = *s & *m; * 'mask->range'.
d++, s++, m++; */
} for (i = 0; i < range_n_bytes(&mask->range); i += sizeof(long))
*d++ = *s++ & *m++;
} }
#define TCP_FLAGS_OFFSET 13 #define TCP_FLAGS_OFFSET 13
...@@ -1000,8 +1006,13 @@ int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key) ...@@ -1000,8 +1006,13 @@ int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key)
static u32 ovs_flow_hash(const struct sw_flow_key *key, int key_start, static u32 ovs_flow_hash(const struct sw_flow_key *key, int key_start,
int key_end) int key_end)
{ {
return jhash2((u32 *)((u8 *)key + key_start), u32 *hash_key = (u32 *)((u8 *)key + key_start);
DIV_ROUND_UP(key_end - key_start, sizeof(u32)), 0); int hash_u32s = (key_end - key_start) >> 2;
/* Make sure number of hash bytes are multiple of u32. */
BUILD_BUG_ON(sizeof(long) % sizeof(u32));
return jhash2(hash_key, hash_u32s, 0);
} }
static int flow_key_start(const struct sw_flow_key *key) static int flow_key_start(const struct sw_flow_key *key)
...@@ -1009,17 +1020,25 @@ static int flow_key_start(const struct sw_flow_key *key) ...@@ -1009,17 +1020,25 @@ static int flow_key_start(const struct sw_flow_key *key)
if (key->tun_key.ipv4_dst) if (key->tun_key.ipv4_dst)
return 0; return 0;
else else
return offsetof(struct sw_flow_key, phy); return rounddown(offsetof(struct sw_flow_key, phy),
sizeof(long));
} }
static bool __cmp_key(const struct sw_flow_key *key1, static bool __cmp_key(const struct sw_flow_key *key1,
const struct sw_flow_key *key2, int key_start, int key_end) const struct sw_flow_key *key2, int key_start, int key_end)
{ {
return !memcmp((u8 *)key1 + key_start, const long *cp1 = (long *)((u8 *)key1 + key_start);
(u8 *)key2 + key_start, (key_end - key_start)); const long *cp2 = (long *)((u8 *)key2 + key_start);
long diffs = 0;
int i;
for (i = key_start; i < key_end; i += sizeof(long))
diffs |= *cp1++ ^ *cp2++;
return diffs == 0;
} }
static bool __flow_cmp_key(const struct sw_flow *flow, static bool __flow_cmp_masked_key(const struct sw_flow *flow,
const struct sw_flow_key *key, int key_start, int key_end) const struct sw_flow_key *key, int key_start, int key_end)
{ {
return __cmp_key(&flow->key, key, key_start, key_end); return __cmp_key(&flow->key, key, key_start, key_end);
...@@ -1056,7 +1075,7 @@ struct sw_flow *ovs_flow_lookup_unmasked_key(struct flow_table *table, ...@@ -1056,7 +1075,7 @@ struct sw_flow *ovs_flow_lookup_unmasked_key(struct flow_table *table,
} }
static struct sw_flow *ovs_masked_flow_lookup(struct flow_table *table, static struct sw_flow *ovs_masked_flow_lookup(struct flow_table *table,
const struct sw_flow_key *flow_key, const struct sw_flow_key *unmasked,
struct sw_flow_mask *mask) struct sw_flow_mask *mask)
{ {
struct sw_flow *flow; struct sw_flow *flow;
...@@ -1066,12 +1085,13 @@ static struct sw_flow *ovs_masked_flow_lookup(struct flow_table *table, ...@@ -1066,12 +1085,13 @@ static struct sw_flow *ovs_masked_flow_lookup(struct flow_table *table,
u32 hash; u32 hash;
struct sw_flow_key masked_key; struct sw_flow_key masked_key;
ovs_flow_key_mask(&masked_key, flow_key, mask); ovs_flow_key_mask(&masked_key, unmasked, mask);
hash = ovs_flow_hash(&masked_key, key_start, key_end); hash = ovs_flow_hash(&masked_key, key_start, key_end);
head = find_bucket(table, hash); head = find_bucket(table, hash);
hlist_for_each_entry_rcu(flow, head, hash_node[table->node_ver]) { hlist_for_each_entry_rcu(flow, head, hash_node[table->node_ver]) {
if (flow->mask == mask && if (flow->mask == mask &&
__flow_cmp_key(flow, &masked_key, key_start, key_end)) __flow_cmp_masked_key(flow, &masked_key,
key_start, key_end))
return flow; return flow;
} }
return NULL; return NULL;
...@@ -1961,6 +1981,8 @@ int ovs_flow_to_nlattrs(const struct sw_flow_key *swkey, ...@@ -1961,6 +1981,8 @@ int ovs_flow_to_nlattrs(const struct sw_flow_key *swkey,
* Returns zero if successful or a negative error code. */ * Returns zero if successful or a negative error code. */
int ovs_flow_init(void) int ovs_flow_init(void)
{ {
BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long));
flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow), 0, flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow), 0,
0, NULL); 0, NULL);
if (flow_cache == NULL) if (flow_cache == NULL)
...@@ -2016,7 +2038,7 @@ static bool ovs_sw_flow_mask_equal(const struct sw_flow_mask *a, ...@@ -2016,7 +2038,7 @@ static bool ovs_sw_flow_mask_equal(const struct sw_flow_mask *a,
return (a->range.end == b->range.end) return (a->range.end == b->range.end)
&& (a->range.start == b->range.start) && (a->range.start == b->range.start)
&& (memcmp(a_, b_, ovs_sw_flow_mask_actual_size(a)) == 0); && (memcmp(a_, b_, range_n_bytes(&a->range)) == 0);
} }
struct sw_flow_mask *ovs_sw_flow_mask_find(const struct flow_table *tbl, struct sw_flow_mask *ovs_sw_flow_mask_find(const struct flow_table *tbl,
...@@ -2053,5 +2075,5 @@ static void ovs_sw_flow_mask_set(struct sw_flow_mask *mask, ...@@ -2053,5 +2075,5 @@ static void ovs_sw_flow_mask_set(struct sw_flow_mask *mask,
u8 *m = (u8 *)&mask->key + range->start; u8 *m = (u8 *)&mask->key + range->start;
mask->range = *range; mask->range = *range;
memset(m, val, ovs_sw_flow_mask_size_roundup(mask)); memset(m, val, range_n_bytes(range));
} }
...@@ -125,7 +125,7 @@ struct sw_flow_key { ...@@ -125,7 +125,7 @@ struct sw_flow_key {
} nd; } nd;
} ipv6; } ipv6;
}; };
}; } __aligned(__alignof__(long));
struct sw_flow { struct sw_flow {
struct rcu_head rcu; struct rcu_head rcu;
...@@ -149,11 +149,6 @@ struct sw_flow_key_range { ...@@ -149,11 +149,6 @@ struct sw_flow_key_range {
size_t end; size_t end;
}; };
static inline u16 ovs_sw_flow_key_range_actual_size(const struct sw_flow_key_range *range)
{
return range->end - range->start;
}
struct sw_flow_match { struct sw_flow_match {
struct sw_flow_key *key; struct sw_flow_key *key;
struct sw_flow_key_range range; struct sw_flow_key_range range;
...@@ -253,18 +248,6 @@ struct sw_flow_mask { ...@@ -253,18 +248,6 @@ struct sw_flow_mask {
struct sw_flow_key key; struct sw_flow_key key;
}; };
static inline u16
ovs_sw_flow_mask_actual_size(const struct sw_flow_mask *mask)
{
return ovs_sw_flow_key_range_actual_size(&mask->range);
}
static inline u16
ovs_sw_flow_mask_size_roundup(const struct sw_flow_mask *mask)
{
return roundup(ovs_sw_flow_mask_actual_size(mask), sizeof(u32));
}
struct sw_flow_mask *ovs_sw_flow_mask_alloc(void); struct sw_flow_mask *ovs_sw_flow_mask_alloc(void);
void ovs_sw_flow_mask_add_ref(struct sw_flow_mask *); void ovs_sw_flow_mask_add_ref(struct sw_flow_mask *);
void ovs_sw_flow_mask_del_ref(struct sw_flow_mask *, bool deferred); void ovs_sw_flow_mask_del_ref(struct sw_flow_mask *, bool deferred);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment