pkt_cls.h 24.1 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5
#ifndef __NET_PKT_CLS_H
#define __NET_PKT_CLS_H

#include <linux/pkt_cls.h>
6
#include <linux/workqueue.h>
Linus Torvalds's avatar
Linus Torvalds committed
7 8
#include <net/sch_generic.h>
#include <net/act_api.h>
9
#include <net/net_namespace.h>
Linus Torvalds's avatar
Linus Torvalds committed
10

11
/* TC action not accessible from user space */
12
#define TC_ACT_CONSUMED		(TC_ACT_VALUE_MAX + 1)
13

Linus Torvalds's avatar
Linus Torvalds committed
14 15
/* Basic packet classifier frontend definitions. */

Eric Dumazet's avatar
Eric Dumazet committed
16
struct tcf_walker {
Linus Torvalds's avatar
Linus Torvalds committed
17 18 19
	int	stop;
	int	skip;
	int	count;
20
	bool	nonempty;
21
	unsigned long cookie;
22
	int	(*fn)(struct tcf_proto *, void *node, struct tcf_walker *);
Linus Torvalds's avatar
Linus Torvalds committed
23 24
};

25 26
int register_tcf_proto_ops(struct tcf_proto_ops *ops);
int unregister_tcf_proto_ops(struct tcf_proto_ops *ops);
Linus Torvalds's avatar
Linus Torvalds committed
27

28
struct tcf_block_ext_info {
29
	enum flow_block_binder_type binder_type;
30 31
	tcf_chain_head_change_t *chain_head_change;
	void *chain_head_change_priv;
32
	u32 block_index;
33 34
};

35 36 37 38 39 40
struct tcf_qevent {
	struct tcf_block	*block;
	struct tcf_block_ext_info info;
	struct tcf_proto __rcu *filter_chain;
};

41
struct tcf_block_cb;
Cong Wang's avatar
Cong Wang committed
42
bool tcf_queue_work(struct rcu_work *rwork, work_func_t func);
43

44
#ifdef CONFIG_NET_CLS
45 46 47
struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block,
				       u32 chain_index);
void tcf_chain_put_by_act(struct tcf_chain *chain);
48 49
struct tcf_chain *tcf_get_next_chain(struct tcf_block *block,
				     struct tcf_chain *chain);
50
struct tcf_proto *tcf_get_next_proto(struct tcf_chain *chain,
51
				     struct tcf_proto *tp);
52
void tcf_block_netif_keep_dst(struct tcf_block *block);
53
int tcf_block_get(struct tcf_block **p_block,
54 55
		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
		  struct netlink_ext_ack *extack);
56
int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
57 58
		      struct tcf_block_ext_info *ei,
		      struct netlink_ext_ack *extack);
59
void tcf_block_put(struct tcf_block *block);
60
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
61
		       struct tcf_block_ext_info *ei);
62

63 64 65 66 67
static inline bool tcf_block_shared(struct tcf_block *block)
{
	return block->index;
}

68 69 70 71 72
static inline bool tcf_block_non_null_shared(struct tcf_block *block)
{
	return block && block->index;
}

73 74
static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
{
75
	WARN_ON(tcf_block_shared(block));
76 77 78
	return block->q;
}

79 80 81 82
int tcf_classify(struct sk_buff *skb,
		 const struct tcf_block *block,
		 const struct tcf_proto *tp, struct tcf_result *res,
		 bool compat_mode);
83

84
#else
85 86 87 88 89
static inline bool tcf_block_shared(struct tcf_block *block)
{
	return false;
}

90 91 92 93 94
static inline bool tcf_block_non_null_shared(struct tcf_block *block)
{
	return false;
}

95 96
static inline
int tcf_block_get(struct tcf_block **p_block,
97 98
		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
		  struct netlink_ext_ack *extack)
99 100 101 102
{
	return 0;
}

103
static inline
104
int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
105 106
		      struct tcf_block_ext_info *ei,
		      struct netlink_ext_ack *extack)
107 108 109 110
{
	return 0;
}

111
static inline void tcf_block_put(struct tcf_block *block)
112 113
{
}
114

115
static inline
116
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
117 118 119 120
		       struct tcf_block_ext_info *ei)
{
}

121 122 123 124 125
static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
{
	return NULL;
}

126
static inline
127
int tc_setup_cb_block_register(struct tcf_block *block, flow_setup_cb_t *cb,
128 129 130 131 132 133
			       void *cb_priv)
{
	return 0;
}

static inline
134
void tc_setup_cb_block_unregister(struct tcf_block *block, flow_setup_cb_t *cb,
135 136 137 138
				  void *cb_priv)
{
}

139 140 141
static inline int tcf_classify(struct sk_buff *skb,
			       const struct tcf_block *block,
			       const struct tcf_proto *tp,
142 143 144 145
			       struct tcf_result *res, bool compat_mode)
{
	return TC_ACT_UNSPEC;
}
146

147
#endif
148

Linus Torvalds's avatar
Linus Torvalds committed
149 150 151
static inline unsigned long
__cls_set_class(unsigned long *clp, unsigned long cl)
{
152
	return xchg(clp, cl);
Linus Torvalds's avatar
Linus Torvalds committed
153 154
}

155 156
static inline void
__tcf_bind_filter(struct Qdisc *q, struct tcf_result *r, unsigned long base)
Linus Torvalds's avatar
Linus Torvalds committed
157
{
158
	unsigned long cl;
159

160 161 162 163
	cl = q->ops->cl_ops->bind_tcf(q, base, r->classid);
	cl = __cls_set_class(&r->class, cl);
	if (cl)
		q->ops->cl_ops->unbind_tcf(q, cl);
Linus Torvalds's avatar
Linus Torvalds committed
164 165 166 167 168
}

static inline void
tcf_bind_filter(struct tcf_proto *tp, struct tcf_result *r, unsigned long base)
{
169
	struct Qdisc *q = tp->chain->block->q;
Linus Torvalds's avatar
Linus Torvalds committed
170

171 172 173 174 175
	/* Check q as it is not set for shared blocks. In that case,
	 * setting class is not supported.
	 */
	if (!q)
		return;
176 177 178 179 180 181 182 183 184 185 186
	sch_tree_lock(q);
	__tcf_bind_filter(q, r, base);
	sch_tree_unlock(q);
}

static inline void
__tcf_unbind_filter(struct Qdisc *q, struct tcf_result *r)
{
	unsigned long cl;

	if ((cl = __cls_set_class(&r->class, 0)) != 0)
187
		q->ops->cl_ops->unbind_tcf(q, cl);
Linus Torvalds's avatar
Linus Torvalds committed
188 189 190 191 192
}

static inline void
tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r)
{
193
	struct Qdisc *q = tp->chain->block->q;
Linus Torvalds's avatar
Linus Torvalds committed
194

195 196
	if (!q)
		return;
197
	__tcf_unbind_filter(q, r);
Linus Torvalds's avatar
Linus Torvalds committed
198 199
}

Eric Dumazet's avatar
Eric Dumazet committed
200
struct tcf_exts {
Linus Torvalds's avatar
Linus Torvalds committed
201
#ifdef CONFIG_NET_CLS_ACT
202
	__u32	type; /* for backward compat(TCA_OLD_COMPAT) */
203 204
	int nr_actions;
	struct tc_action **actions;
205 206
	struct net	*net;
	netns_tracker	ns_tracker;
Linus Torvalds's avatar
Linus Torvalds committed
207
#endif
208 209 210
	/* Map to export classifier specific extension TLV types to the
	 * generic extensions API. Unsupported extensions must be set to 0.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
211 212 213 214
	int action;
	int police;
};

215 216
static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net,
				int action, int police)
217 218
{
#ifdef CONFIG_NET_CLS_ACT
219
	exts->type = 0;
220
	exts->nr_actions = 0;
221
	exts->net = net;
222
	netns_tracker_alloc(net, &exts->ns_tracker, GFP_KERNEL);
223 224
	exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *),
				GFP_KERNEL);
225 226
	if (!exts->actions)
		return -ENOMEM;
227
#endif
228 229
	exts->action = action;
	exts->police = police;
230
	return 0;
231 232
}

233 234 235 236 237 238 239 240
/* Return false if the netns is being destroyed in cleanup_net(). Callers
 * need to do cleanup synchronously in this case, otherwise may race with
 * tc_action_net_exit(). Return true for other cases.
 */
static inline bool tcf_exts_get_net(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
	exts->net = maybe_get_net(exts->net);
241 242
	if (exts->net)
		netns_tracker_alloc(exts->net, &exts->ns_tracker, GFP_KERNEL);
243 244 245 246 247 248 249 250 251 252
	return exts->net != NULL;
#else
	return true;
#endif
}

static inline void tcf_exts_put_net(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
	if (exts->net)
253
		put_net_track(exts->net, &exts->ns_tracker);
254 255 256
#endif
}

257
#ifdef CONFIG_NET_CLS_ACT
258 259 260 261
#define tcf_exts_for_each_action(i, a, exts) \
	for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = (exts)->actions[i]); i++)
#else
#define tcf_exts_for_each_action(i, a, exts) \
262
	for (; 0; (void)(i), (void)(a), (void)(exts))
263 264
#endif

265 266
static inline void
tcf_exts_stats_update(const struct tcf_exts *exts,
267
		      u64 bytes, u64 packets, u64 drops, u64 lastuse,
268
		      u8 used_hw_stats, bool used_hw_stats_valid)
269 270 271 272 273 274 275 276 277
{
#ifdef CONFIG_NET_CLS_ACT
	int i;

	preempt_disable();

	for (i = 0; i < exts->nr_actions; i++) {
		struct tc_action *a = exts->actions[i];

278 279
		tcf_action_stats_update(a, bytes, packets, drops,
					lastuse, true);
280 281
		a->used_hw_stats = used_hw_stats;
		a->used_hw_stats_valid = used_hw_stats_valid;
282 283 284 285 286 287
	}

	preempt_enable();
#endif
}

288 289 290 291 292 293 294 295
/**
 * tcf_exts_has_actions - check if at least one action is present
 * @exts: tc filter extensions handle
 *
 * Returns true if at least one action is present.
 */
static inline bool tcf_exts_has_actions(struct tcf_exts *exts)
{
296
#ifdef CONFIG_NET_CLS_ACT
297 298 299 300 301
	return exts->nr_actions;
#else
	return false;
#endif
}
302

303 304 305 306 307 308
/**
 * tcf_exts_exec - execute tc filter extensions
 * @skb: socket buffer
 * @exts: tc filter extensions handle
 * @res: desired result
 *
309
 * Executes all configured extensions. Returns TC_ACT_OK on a normal execution,
310 311 312 313 314 315 316 317 318
 * a negative number if the filter must be considered unmatched or
 * a positive action code (TC_ACT_*) which must be returned to the
 * underlying layer.
 */
static inline int
tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts,
	      struct tcf_result *res)
{
#ifdef CONFIG_NET_CLS_ACT
319
	return tcf_action_exec(skb, exts->actions, exts->nr_actions, res);
320
#endif
321
	return TC_ACT_OK;
322 323
}

324 325
int tcf_exts_validate(struct net *net, struct tcf_proto *tp,
		      struct nlattr **tb, struct nlattr *rate_tlv,
326
		      struct tcf_exts *exts, u32 flags,
327
		      struct netlink_ext_ack *extack);
328
void tcf_exts_destroy(struct tcf_exts *exts);
329
void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src);
330
int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts);
331
int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts);
332
int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts);
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335

/**
 * struct tcf_pkt_info - packet information
336 337 338
 *
 * @ptr: start of the pkt data
 * @nexthdr: offset of the next header
Linus Torvalds's avatar
Linus Torvalds committed
339
 */
Eric Dumazet's avatar
Eric Dumazet committed
340
struct tcf_pkt_info {
Linus Torvalds's avatar
Linus Torvalds committed
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
	unsigned char *		ptr;
	int			nexthdr;
};

#ifdef CONFIG_NET_EMATCH

struct tcf_ematch_ops;

/**
 * struct tcf_ematch - extended match (ematch)
 * 
 * @matchid: identifier to allow userspace to reidentify a match
 * @flags: flags specifying attributes and the relation to other matches
 * @ops: the operations lookup table of the corresponding ematch module
 * @datalen: length of the ematch specific configuration data
 * @data: ematch specific data
357
 * @net: the network namespace
Linus Torvalds's avatar
Linus Torvalds committed
358
 */
Eric Dumazet's avatar
Eric Dumazet committed
359
struct tcf_ematch {
Linus Torvalds's avatar
Linus Torvalds committed
360 361 362 363 364
	struct tcf_ematch_ops * ops;
	unsigned long		data;
	unsigned int		datalen;
	u16			matchid;
	u16			flags;
365
	struct net		*net;
Linus Torvalds's avatar
Linus Torvalds committed
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
};

static inline int tcf_em_is_container(struct tcf_ematch *em)
{
	return !em->ops;
}

static inline int tcf_em_is_simple(struct tcf_ematch *em)
{
	return em->flags & TCF_EM_SIMPLE;
}

static inline int tcf_em_is_inverted(struct tcf_ematch *em)
{
	return em->flags & TCF_EM_INVERT;
}

static inline int tcf_em_last_match(struct tcf_ematch *em)
{
	return (em->flags & TCF_EM_REL_MASK) == TCF_EM_REL_END;
}

static inline int tcf_em_early_end(struct tcf_ematch *em, int result)
{
	if (tcf_em_last_match(em))
		return 1;

	if (result == 0 && em->flags & TCF_EM_REL_AND)
		return 1;

	if (result != 0 && em->flags & TCF_EM_REL_OR)
		return 1;

	return 0;
}
	
/**
 * struct tcf_ematch_tree - ematch tree handle
 *
 * @hdr: ematch tree header supplied by userspace
 * @matches: array of ematches
 */
Eric Dumazet's avatar
Eric Dumazet committed
408
struct tcf_ematch_tree {
Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
	struct tcf_ematch_tree_hdr hdr;
	struct tcf_ematch *	matches;
	
};

/**
 * struct tcf_ematch_ops - ematch module operations
 * 
 * @kind: identifier (kind) of this ematch module
 * @datalen: length of expected configuration data (optional)
 * @change: called during validation (optional)
 * @match: called during ematch tree evaluation, must return 1/0
 * @destroy: called during destroyage (optional)
 * @dump: called during dumping process (optional)
 * @owner: owner, must be set to THIS_MODULE
 * @link: link to previous/next ematch module (internal use)
 */
Eric Dumazet's avatar
Eric Dumazet committed
426
struct tcf_ematch_ops {
Linus Torvalds's avatar
Linus Torvalds committed
427 428
	int			kind;
	int			datalen;
429
	int			(*change)(struct net *net, void *,
Linus Torvalds's avatar
Linus Torvalds committed
430 431 432
					  int, struct tcf_ematch *);
	int			(*match)(struct sk_buff *, struct tcf_ematch *,
					 struct tcf_pkt_info *);
433
	void			(*destroy)(struct tcf_ematch *);
Linus Torvalds's avatar
Linus Torvalds committed
434 435 436 437 438
	int			(*dump)(struct sk_buff *, struct tcf_ematch *);
	struct module		*owner;
	struct list_head	link;
};

439 440 441 442
int tcf_em_register(struct tcf_ematch_ops *);
void tcf_em_unregister(struct tcf_ematch_ops *);
int tcf_em_tree_validate(struct tcf_proto *, struct nlattr *,
			 struct tcf_ematch_tree *);
443
void tcf_em_tree_destroy(struct tcf_ematch_tree *);
444 445 446
int tcf_em_tree_dump(struct sk_buff *, struct tcf_ematch_tree *, int);
int __tcf_em_tree_match(struct sk_buff *, struct tcf_ematch_tree *,
			struct tcf_pkt_info *);
Linus Torvalds's avatar
Linus Torvalds committed
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471

/**
 * tcf_em_tree_match - evaulate an ematch tree
 *
 * @skb: socket buffer of the packet in question
 * @tree: ematch tree to be used for evaluation
 * @info: packet information examined by classifier
 *
 * This function matches @skb against the ematch tree in @tree by going
 * through all ematches respecting their logic relations returning
 * as soon as the result is obvious.
 *
 * Returns 1 if the ematch tree as-one matches, no ematches are configured
 * or ematch is not enabled in the kernel, otherwise 0 is returned.
 */
static inline int tcf_em_tree_match(struct sk_buff *skb,
				    struct tcf_ematch_tree *tree,
				    struct tcf_pkt_info *info)
{
	if (tree->hdr.nmatches)
		return __tcf_em_tree_match(skb, tree, info);
	else
		return 1;
}

472 473
#define MODULE_ALIAS_TCF_EMATCH(kind)	MODULE_ALIAS("ematch-kind-" __stringify(kind))

Linus Torvalds's avatar
Linus Torvalds committed
474 475
#else /* CONFIG_NET_EMATCH */

Eric Dumazet's avatar
Eric Dumazet committed
476
struct tcf_ematch_tree {
Linus Torvalds's avatar
Linus Torvalds committed
477 478 479
};

#define tcf_em_tree_validate(tp, tb, t) ((void)(t), 0)
480
#define tcf_em_tree_destroy(t) do { (void)(t); } while(0)
Linus Torvalds's avatar
Linus Torvalds committed
481 482 483 484 485 486 487 488 489
#define tcf_em_tree_dump(skb, t, tlv) (0)
#define tcf_em_tree_match(skb, t, info) ((void)(info), 1)

#endif /* CONFIG_NET_EMATCH */

static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer)
{
	switch (layer) {
		case TCF_LAYER_LINK:
490
			return skb_mac_header(skb);
Linus Torvalds's avatar
Linus Torvalds committed
491
		case TCF_LAYER_NETWORK:
492
			return skb_network_header(skb);
Linus Torvalds's avatar
Linus Torvalds committed
493
		case TCF_LAYER_TRANSPORT:
494
			return skb_transport_header(skb);
Linus Torvalds's avatar
Linus Torvalds committed
495 496 497 498 499
	}

	return NULL;
}

500 501
static inline int tcf_valid_offset(const struct sk_buff *skb,
				   const unsigned char *ptr, const int len)
Linus Torvalds's avatar
Linus Torvalds committed
502
{
503 504 505
	return likely((ptr + len) <= skb_tail_pointer(skb) &&
		      ptr >= skb->head &&
		      (ptr <= (ptr + len)));
Linus Torvalds's avatar
Linus Torvalds committed
506 507 508
}

static inline int
509 510
tcf_change_indev(struct net *net, struct nlattr *indev_tlv,
		 struct netlink_ext_ack *extack)
Linus Torvalds's avatar
Linus Torvalds committed
511
{
512 513 514
	char indev[IFNAMSIZ];
	struct net_device *dev;

515
	if (nla_strscpy(indev, indev_tlv, IFNAMSIZ) < 0) {
516 517
		NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
				    "Interface name too long");
Linus Torvalds's avatar
Linus Torvalds committed
518
		return -EINVAL;
519
	}
520
	dev = __dev_get_by_name(net, indev);
521 522 523
	if (!dev) {
		NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
				    "Network device not found");
524
		return -ENODEV;
525
	}
526
	return dev->ifindex;
Linus Torvalds's avatar
Linus Torvalds committed
527 528
}

529 530
static inline bool
tcf_match_indev(struct sk_buff *skb, int ifindex)
Linus Torvalds's avatar
Linus Torvalds committed
531
{
532 533 534 535 536
	if (!ifindex)
		return true;
	if  (!skb->skb_iif)
		return false;
	return ifindex == skb->skb_iif;
Linus Torvalds's avatar
Linus Torvalds committed
537 538
}

539
int tc_setup_flow_action(struct flow_action *flow_action,
540
			 const struct tcf_exts *exts);
541 542
void tc_cleanup_flow_action(struct flow_action *flow_action);

543
int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
		     void *type_data, bool err_stop, bool rtnl_held);
int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
		    enum tc_setup_type type, void *type_data, bool err_stop,
		    u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
			enum tc_setup_type type, void *type_data, bool err_stop,
			u32 *old_flags, unsigned int *old_in_hw_count,
			u32 *new_flags, unsigned int *new_in_hw_count,
			bool rtnl_held);
int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
			enum tc_setup_type type, void *type_data, bool err_stop,
			u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
			  bool add, flow_setup_cb_t *cb,
			  enum tc_setup_type type, void *type_data,
			  void *cb_priv, u32 *flags, unsigned int *in_hw_count);
560
unsigned int tcf_exts_num_actions(struct tcf_exts *exts);
561

562 563 564 565 566 567 568 569 570
#ifdef CONFIG_NET_CLS_ACT
int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
		    enum flow_block_binder_type binder_type,
		    struct nlattr *block_index_attr,
		    struct netlink_ext_ack *extack);
void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch);
int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
			       struct netlink_ext_ack *extack);
struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
571
				  struct sk_buff **to_free, int *ret);
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe);
#else
static inline int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
				  enum flow_block_binder_type binder_type,
				  struct nlattr *block_index_attr,
				  struct netlink_ext_ack *extack)
{
	return 0;
}

static inline void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
{
}

static inline int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
					     struct netlink_ext_ack *extack)
{
	return 0;
}

static inline struct sk_buff *
tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
594
		  struct sk_buff **to_free, int *ret)
595 596 597 598 599 600 601 602 603 604
{
	return skb;
}

static inline int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
{
	return 0;
}
#endif

605 606
struct tc_cls_u32_knode {
	struct tcf_exts *exts;
607
	struct tcf_result *res;
608
	struct tc_u32_sel *sel;
609 610 611 612
	u32 handle;
	u32 val;
	u32 mask;
	u32 link_handle;
613
	u8 fshift;
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
};

struct tc_cls_u32_hnode {
	u32 handle;
	u32 prio;
	unsigned int divisor;
};

enum tc_clsu32_command {
	TC_CLSU32_NEW_KNODE,
	TC_CLSU32_REPLACE_KNODE,
	TC_CLSU32_DELETE_KNODE,
	TC_CLSU32_NEW_HNODE,
	TC_CLSU32_REPLACE_HNODE,
	TC_CLSU32_DELETE_HNODE,
};

struct tc_cls_u32_offload {
632
	struct flow_cls_common_offload common;
633 634 635 636 637 638 639 640
	/* knode values */
	enum tc_clsu32_command command;
	union {
		struct tc_cls_u32_knode knode;
		struct tc_cls_u32_hnode hnode;
	};
};

641
static inline bool tc_can_offload(const struct net_device *dev)
642
{
643
	return dev->features & NETIF_F_HW_TC;
644 645
}

646 647 648 649 650 651 652 653 654 655 656
static inline bool tc_can_offload_extack(const struct net_device *dev,
					 struct netlink_ext_ack *extack)
{
	bool can = tc_can_offload(dev);

	if (!can)
		NL_SET_ERR_MSG(extack, "TC offload is disabled on net device");

	return can;
}

657 658
static inline bool
tc_cls_can_offload_and_chain0(const struct net_device *dev,
659
			      struct flow_cls_common_offload *common)
660 661 662 663 664 665 666 667 668 669 670
{
	if (!tc_can_offload_extack(dev, common->extack))
		return false;
	if (common->chain_index) {
		NL_SET_ERR_MSG(common->extack,
			       "Driver supports only offload of chain 0");
		return false;
	}
	return true;
}

671 672 673 674 675
static inline bool tc_skip_hw(u32 flags)
{
	return (flags & TCA_CLS_FLAGS_SKIP_HW) ? true : false;
}

676 677 678 679 680 681 682 683
static inline bool tc_skip_sw(u32 flags)
{
	return (flags & TCA_CLS_FLAGS_SKIP_SW) ? true : false;
}

/* SKIP_HW and SKIP_SW are mutually exclusive flags. */
static inline bool tc_flags_valid(u32 flags)
{
684 685
	if (flags & ~(TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW |
		      TCA_CLS_FLAGS_VERBOSE))
686 687
		return false;

688
	flags &= TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW;
689 690 691 692 693 694
	if (!(flags ^ (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)))
		return false;

	return true;
}

695 696 697 698 699
static inline bool tc_in_hw(u32 flags)
{
	return (flags & TCA_CLS_FLAGS_IN_HW) ? true : false;
}

700
static inline void
701
tc_cls_common_offload_init(struct flow_cls_common_offload *cls_common,
702 703 704 705 706
			   const struct tcf_proto *tp, u32 flags,
			   struct netlink_ext_ack *extack)
{
	cls_common->chain_index = tp->chain->index;
	cls_common->protocol = tp->protocol;
707
	cls_common->prio = tp->prio >> 16;
708
	if (tc_skip_sw(flags) || flags & TCA_CLS_FLAGS_VERBOSE)
709 710 711
		cls_common->extack = extack;
}

712 713 714 715 716 717 718 719 720 721 722
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static inline struct tc_skb_ext *tc_skb_ext_alloc(struct sk_buff *skb)
{
	struct tc_skb_ext *tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);

	if (tc_skb_ext)
		memset(tc_skb_ext, 0, sizeof(*tc_skb_ext));
	return tc_skb_ext;
}
#endif

723 724 725
enum tc_matchall_command {
	TC_CLSMATCHALL_REPLACE,
	TC_CLSMATCHALL_DESTROY,
726
	TC_CLSMATCHALL_STATS,
727 728 729
};

struct tc_cls_matchall_offload {
730
	struct flow_cls_common_offload common;
731
	enum tc_matchall_command command;
732
	struct flow_rule *rule;
733
	struct flow_stats stats;
734 735 736
	unsigned long cookie;
};

737
enum tc_clsbpf_command {
738
	TC_CLSBPF_OFFLOAD,
739
	TC_CLSBPF_STATS,
740 741 742
};

struct tc_cls_bpf_offload {
743
	struct flow_cls_common_offload common;
744 745 746
	enum tc_clsbpf_command command;
	struct tcf_exts *exts;
	struct bpf_prog *prog;
747
	struct bpf_prog *oldprog;
748 749 750 751
	const char *name;
	bool exts_integrated;
};

752 753 754 755 756 757 758 759 760
struct tc_mqprio_qopt_offload {
	/* struct tc_mqprio_qopt must always be the first element */
	struct tc_mqprio_qopt qopt;
	u16 mode;
	u16 shaper;
	u32 flags;
	u64 min_rate[TC_QOPT_MAX_QUEUE];
	u64 max_rate[TC_QOPT_MAX_QUEUE];
};
761 762 763 764 765 766 767

/* This structure holds cookie structure that is passed from user
 * to the kernel for actions and classifiers
 */
struct tc_cookie {
	u8  *data;
	u32 len;
768
	struct rcu_head rcu;
769
};
770

771
struct tc_qopt_offload_stats {
772
	struct gnet_stats_basic_sync *bstats;
773 774 775
	struct gnet_stats_queue *qstats;
};

776 777 778
enum tc_mq_command {
	TC_MQ_CREATE,
	TC_MQ_DESTROY,
779
	TC_MQ_STATS,
780 781 782 783 784 785
	TC_MQ_GRAFT,
};

struct tc_mq_opt_offload_graft_params {
	unsigned long queue;
	u32 child_handle;
786 787 788 789 790
};

struct tc_mq_qopt_offload {
	enum tc_mq_command command;
	u32 handle;
791 792 793 794
	union {
		struct tc_qopt_offload_stats stats;
		struct tc_mq_opt_offload_graft_params graft_params;
	};
795 796
};

797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
enum tc_htb_command {
	/* Root */
	TC_HTB_CREATE, /* Initialize HTB offload. */
	TC_HTB_DESTROY, /* Destroy HTB offload. */

	/* Classes */
	/* Allocate qid and create leaf. */
	TC_HTB_LEAF_ALLOC_QUEUE,
	/* Convert leaf to inner, preserve and return qid, create new leaf. */
	TC_HTB_LEAF_TO_INNER,
	/* Delete leaf, while siblings remain. */
	TC_HTB_LEAF_DEL,
	/* Delete leaf, convert parent to leaf, preserving qid. */
	TC_HTB_LEAF_DEL_LAST,
	/* TC_HTB_LEAF_DEL_LAST, but delete driver data on hardware errors. */
	TC_HTB_LEAF_DEL_LAST_FORCE,
	/* Modify parameters of a node. */
	TC_HTB_NODE_MODIFY,

	/* Class qdisc */
	TC_HTB_LEAF_QUERY_QUEUE, /* Query qid by classid. */
};

struct tc_htb_qopt_offload {
	struct netlink_ext_ack *extack;
	enum tc_htb_command command;
	u32 parent_classid;
824
	u16 classid;
825 826 827 828 829 830 831
	u16 qid;
	u64 rate;
	u64 ceil;
};

#define TC_HTB_CLASSID_ROOT U32_MAX

832 833 834 835 836
enum tc_red_command {
	TC_RED_REPLACE,
	TC_RED_DESTROY,
	TC_RED_STATS,
	TC_RED_XSTATS,
837
	TC_RED_GRAFT,
838 839 840 841 842 843
};

struct tc_red_qopt_offload_params {
	u32 min;
	u32 max;
	u32 probability;
844
	u32 limit;
845
	bool is_ecn;
846
	bool is_harddrop;
847
	bool is_nodrop;
848
	struct gnet_stats_queue *qstats;
849 850 851 852 853 854 855 856
};

struct tc_red_qopt_offload {
	enum tc_red_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_red_qopt_offload_params set;
857
		struct tc_qopt_offload_stats stats;
858
		struct red_stats *xstats;
859
		u32 child_handle;
860 861 862
	};
};

863 864 865
enum tc_gred_command {
	TC_GRED_REPLACE,
	TC_GRED_DESTROY,
866
	TC_GRED_STATS,
867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
};

struct tc_gred_vq_qopt_offload_params {
	bool present;
	u32 limit;
	u32 prio;
	u32 min;
	u32 max;
	bool is_ecn;
	bool is_harddrop;
	u32 probability;
	/* Only need backlog, see struct tc_prio_qopt_offload_params */
	u32 *backlog;
};

struct tc_gred_qopt_offload_params {
	bool grio_on;
	bool wred_on;
	unsigned int dp_cnt;
	unsigned int dp_def;
	struct gnet_stats_queue *qstats;
	struct tc_gred_vq_qopt_offload_params tab[MAX_DPs];
};

891
struct tc_gred_qopt_offload_stats {
892
	struct gnet_stats_basic_sync bstats[MAX_DPs];
893 894 895 896
	struct gnet_stats_queue qstats[MAX_DPs];
	struct red_stats *xstats[MAX_DPs];
};

897 898 899 900 901 902
struct tc_gred_qopt_offload {
	enum tc_gred_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_gred_qopt_offload_params set;
903
		struct tc_gred_qopt_offload_stats stats;
904 905 906
	};
};

907 908 909 910
enum tc_prio_command {
	TC_PRIO_REPLACE,
	TC_PRIO_DESTROY,
	TC_PRIO_STATS,
911
	TC_PRIO_GRAFT,
912 913 914 915 916
};

struct tc_prio_qopt_offload_params {
	int bands;
	u8 priomap[TC_PRIO_MAX + 1];
917 918
	/* At the point of un-offloading the Qdisc, the reported backlog and
	 * qlen need to be reduced by the portion that is in HW.
919 920 921 922
	 */
	struct gnet_stats_queue *qstats;
};

923 924 925 926 927
struct tc_prio_qopt_offload_graft_params {
	u8 band;
	u32 child_handle;
};

928 929 930 931 932 933 934
struct tc_prio_qopt_offload {
	enum tc_prio_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_prio_qopt_offload_params replace_params;
		struct tc_qopt_offload_stats stats;
935
		struct tc_prio_qopt_offload_graft_params graft_params;
936 937
	};
};
938

939 940 941 942 943 944 945 946 947 948
enum tc_root_command {
	TC_ROOT_GRAFT,
};

struct tc_root_qopt_offload {
	enum tc_root_command command;
	u32 handle;
	bool ingress;
};

949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
enum tc_ets_command {
	TC_ETS_REPLACE,
	TC_ETS_DESTROY,
	TC_ETS_STATS,
	TC_ETS_GRAFT,
};

struct tc_ets_qopt_offload_replace_params {
	unsigned int bands;
	u8 priomap[TC_PRIO_MAX + 1];
	unsigned int quanta[TCQ_ETS_MAX_BANDS];	/* 0 for strict bands. */
	unsigned int weights[TCQ_ETS_MAX_BANDS];
	struct gnet_stats_queue *qstats;
};

struct tc_ets_qopt_offload_graft_params {
	u8 band;
	u32 child_handle;
};

struct tc_ets_qopt_offload {
	enum tc_ets_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_ets_qopt_offload_replace_params replace_params;
		struct tc_qopt_offload_stats stats;
		struct tc_ets_qopt_offload_graft_params graft_params;
	};
};

980 981 982 983
enum tc_tbf_command {
	TC_TBF_REPLACE,
	TC_TBF_DESTROY,
	TC_TBF_STATS,
984
	TC_TBF_GRAFT,
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
};

struct tc_tbf_qopt_offload_replace_params {
	struct psched_ratecfg rate;
	u32 max_size;
	struct gnet_stats_queue *qstats;
};

struct tc_tbf_qopt_offload {
	enum tc_tbf_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_tbf_qopt_offload_replace_params replace_params;
		struct tc_qopt_offload_stats stats;
1000
		u32 child_handle;
1001 1002 1003
	};
};

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
enum tc_fifo_command {
	TC_FIFO_REPLACE,
	TC_FIFO_DESTROY,
	TC_FIFO_STATS,
};

struct tc_fifo_qopt_offload {
	enum tc_fifo_command command;
	u32 handle;
	u32 parent;
	union {
		struct tc_qopt_offload_stats stats;
	};
};

Linus Torvalds's avatar
Linus Torvalds committed
1019
#endif