Merge branch 'tools-ynl-generate-code-for-the-devlink-family'

Jakub Kicinski says:

====================
tools: ynl: generate code for the devlink family

Another chunk of changes to support more capabilities in the YNL
code gen. Devlink brings in deep nesting and directional messages
(requests and responses have different IDs). We need a healthy
dose of codegen changes to support those (I wasn't planning to
support code gen for "directional" families initially, but
the importance of devlink and ethtool is undeniable).
====================

Link: https://lore.kernel.org/r/20230607202403.1089925-1-kuba@kernel.orgSigned-off-by: Jakub Kicinski <kuba@kernel.org>

Documentation/netlink/specs/devlink.yaml

@@ -9,6 +9,7 @@ doc: Partial family for Devlink.
 attribute-sets:
   -
     name: devlink
+    name-prefix: devlink-attr-
     attributes:
       -
         name: bus-name
@@ -95,10 +96,12 @@ attribute-sets:
       -
         name: reload-action-info
         type: nest
+        multi-attr: true
         nested-attributes: dl-reload-act-info
       -
         name: reload-action-stats
         type: nest
+        multi-attr: true
         nested-attributes: dl-reload-act-stats
   -
     name: dl-dev-stats
@@ -196,3 +199,8 @@ operations:
           attributes:
             - bus-name
             - dev-name
+            - info-driver-name
+            - info-serial-number
+            - info-version-fixed
+            - info-version-running
+            - info-version-stored

tools/net/ynl/generated/Makefile

@@ -9,7 +9,7 @@ endif
 
 TOOL:=../ynl-gen-c.py
 
-GENS:=handshake fou netdev
+GENS:=devlink handshake fou netdev
 SRCS=$(patsubst %,%-user.c,${GENS})
 HDRS=$(patsubst %,%-user.h,${GENS})
 OBJS=$(patsubst %,%-user.o,${GENS})

tools/net/ynl/generated/devlink-user.h

+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/devlink.yaml */
+/* YNL-GEN user header */
+
+#ifndef _LINUX_DEVLINK_GEN_H
+#define _LINUX_DEVLINK_GEN_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <linux/types.h>
+#include <linux/devlink.h>
+
+struct ynl_sock;
+
+extern const struct ynl_family ynl_devlink_family;
+
+/* Enums */
+const char *devlink_op_str(int op);
+
+/* Common nested types */
+struct devlink_dl_info_version {
+	struct {
+		__u32 info_version_name_len;
+		__u32 info_version_value_len;
+	} _present;
+
+	char *info_version_name;
+	char *info_version_value;
+};
+
+struct devlink_dl_reload_stats_entry {
+	struct {
+		__u32 reload_stats_limit:1;
+		__u32 reload_stats_value:1;
+	} _present;
+
+	__u8 reload_stats_limit;
+	__u32 reload_stats_value;
+};
+
+struct devlink_dl_reload_act_stats {
+	unsigned int n_reload_stats_entry;
+	struct devlink_dl_reload_stats_entry *reload_stats_entry;
+};
+
+struct devlink_dl_reload_act_info {
+	struct {
+		__u32 reload_action:1;
+	} _present;
+
+	__u8 reload_action;
+	unsigned int n_reload_action_stats;
+	struct devlink_dl_reload_act_stats *reload_action_stats;
+};
+
+struct devlink_dl_reload_stats {
+	unsigned int n_reload_action_info;
+	struct devlink_dl_reload_act_info *reload_action_info;
+};
+
+struct devlink_dl_dev_stats {
+	struct {
+		__u32 reload_stats:1;
+		__u32 remote_reload_stats:1;
+	} _present;
+
+	struct devlink_dl_reload_stats reload_stats;
+	struct devlink_dl_reload_stats remote_reload_stats;
+};
+
+/* ============== DEVLINK_CMD_GET ============== */
+/* DEVLINK_CMD_GET - do */
+struct devlink_get_req {
+	struct {
+		__u32 bus_name_len;
+		__u32 dev_name_len;
+	} _present;
+
+	char *bus_name;
+	char *dev_name;
+};
+
+static inline struct devlink_get_req *devlink_get_req_alloc(void)
+{
+	return calloc(1, sizeof(struct devlink_get_req));
+}
+void devlink_get_req_free(struct devlink_get_req *req);
+
+static inline void
+devlink_get_req_set_bus_name(struct devlink_get_req *req, const char *bus_name)
+{
+	free(req->bus_name);
+	req->_present.bus_name_len = strlen(bus_name);
+	req->bus_name = malloc(req->_present.bus_name_len + 1);
+	memcpy(req->bus_name, bus_name, req->_present.bus_name_len);
+	req->bus_name[req->_present.bus_name_len] = 0;
+}
+static inline void
+devlink_get_req_set_dev_name(struct devlink_get_req *req, const char *dev_name)
+{
+	free(req->dev_name);
+	req->_present.dev_name_len = strlen(dev_name);
+	req->dev_name = malloc(req->_present.dev_name_len + 1);
+	memcpy(req->dev_name, dev_name, req->_present.dev_name_len);
+	req->dev_name[req->_present.dev_name_len] = 0;
+}
+
+struct devlink_get_rsp {
+	struct {
+		__u32 bus_name_len;
+		__u32 dev_name_len;
+		__u32 reload_failed:1;
+		__u32 reload_action:1;
+		__u32 dev_stats:1;
+	} _present;
+
+	char *bus_name;
+	char *dev_name;
+	__u8 reload_failed;
+	__u8 reload_action;
+	struct devlink_dl_dev_stats dev_stats;
+};
+
+void devlink_get_rsp_free(struct devlink_get_rsp *rsp);
+
+/*
+ * Get devlink instances.
+ */
+struct devlink_get_rsp *
+devlink_get(struct ynl_sock *ys, struct devlink_get_req *req);
+
+/* DEVLINK_CMD_GET - dump */
+struct devlink_get_list {
+	struct devlink_get_list *next;
+	struct devlink_get_rsp obj __attribute__ ((aligned (8)));
+};
+
+void devlink_get_list_free(struct devlink_get_list *rsp);
+
+struct devlink_get_list *devlink_get_dump(struct ynl_sock *ys);
+
+/* ============== DEVLINK_CMD_INFO_GET ============== */
+/* DEVLINK_CMD_INFO_GET - do */
+struct devlink_info_get_req {
+	struct {
+		__u32 bus_name_len;
+		__u32 dev_name_len;
+	} _present;
+
+	char *bus_name;
+	char *dev_name;
+};
+
+static inline struct devlink_info_get_req *devlink_info_get_req_alloc(void)
+{
+	return calloc(1, sizeof(struct devlink_info_get_req));
+}
+void devlink_info_get_req_free(struct devlink_info_get_req *req);
+
+static inline void
+devlink_info_get_req_set_bus_name(struct devlink_info_get_req *req,
+				  const char *bus_name)
+{
+	free(req->bus_name);
+	req->_present.bus_name_len = strlen(bus_name);
+	req->bus_name = malloc(req->_present.bus_name_len + 1);
+	memcpy(req->bus_name, bus_name, req->_present.bus_name_len);
+	req->bus_name[req->_present.bus_name_len] = 0;
+}
+static inline void
+devlink_info_get_req_set_dev_name(struct devlink_info_get_req *req,
+				  const char *dev_name)
+{
+	free(req->dev_name);
+	req->_present.dev_name_len = strlen(dev_name);
+	req->dev_name = malloc(req->_present.dev_name_len + 1);
+	memcpy(req->dev_name, dev_name, req->_present.dev_name_len);
+	req->dev_name[req->_present.dev_name_len] = 0;
+}
+
+struct devlink_info_get_rsp {
+	struct {
+		__u32 bus_name_len;
+		__u32 dev_name_len;
+		__u32 info_driver_name_len;
+		__u32 info_serial_number_len;
+	} _present;
+
+	char *bus_name;
+	char *dev_name;
+	char *info_driver_name;
+	char *info_serial_number;
+	unsigned int n_info_version_fixed;
+	struct devlink_dl_info_version *info_version_fixed;
+	unsigned int n_info_version_running;
+	struct devlink_dl_info_version *info_version_running;
+	unsigned int n_info_version_stored;
+	struct devlink_dl_info_version *info_version_stored;
+};
+
+void devlink_info_get_rsp_free(struct devlink_info_get_rsp *rsp);
+
+/*
+ * Get device information, like driver name, hardware and firmware versions etc.
+ */
+struct devlink_info_get_rsp *
+devlink_info_get(struct ynl_sock *ys, struct devlink_info_get_req *req);
+
+#endif /* _LINUX_DEVLINK_GEN_H */

tools/net/ynl/generated/fou-user.c

@@ -16,7 +16,6 @@
 
 /* Enums */
 static const char * const fou_op_strmap[] = {
-	[FOU_CMD_UNSPEC] = "unspec",
 	[FOU_CMD_ADD] = "add",
 	[FOU_CMD_DEL] = "del",
 	[FOU_CMD_GET] = "get",
@@ -43,8 +42,6 @@ const char *fou_encap_type_str(int value)
 }
 
 /* Policies */
-extern struct ynl_policy_nest fou_nest;
-
 struct ynl_policy_attr fou_policy[FOU_ATTR_MAX + 1] = {
 	[FOU_ATTR_UNSPEC] = { .name = "unspec", .type = YNL_PT_REJECT, },
 	[FOU_ATTR_PORT] = { .name = "port", .type = YNL_PT_U16, },

tools/net/ynl/generated/handshake-user.c

@@ -69,10 +69,6 @@ const char *handshake_auth_str(enum handshake_auth value)
 }
 
 /* Policies */
-extern struct ynl_policy_nest handshake_x509_nest;
-extern struct ynl_policy_nest handshake_accept_nest;
-extern struct ynl_policy_nest handshake_done_nest;
-
 struct ynl_policy_attr handshake_x509_policy[HANDSHAKE_A_X509_MAX + 1] = {
 	[HANDSHAKE_A_X509_CERT] = { .name = "cert", .type = YNL_PT_U32, },
 	[HANDSHAKE_A_X509_PRIVKEY] = { .name = "privkey", .type = YNL_PT_U32, },

tools/net/ynl/generated/netdev-user.c

@@ -48,8 +48,6 @@ const char *netdev_xdp_act_str(enum netdev_xdp_act value)
 }
 
 /* Policies */
-extern struct ynl_policy_nest netdev_dev_nest;
-
 struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = {
 	[NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, },
 	[NETDEV_A_DEV_PAD] = { .name = "pad", .type = YNL_PT_IGNORE, },

tools/net/ynl/lib/nlspec.py

@@ -324,6 +324,7 @@ class SpecFamily(SpecElement):
 
     Attributes:
         proto     protocol type (e.g. genetlink)
+        msg_id_model   enum-model for operations (unified, directional etc.)
         license   spec license (loaded from an SPDX tag on the spec)
 
         attr_sets  dict of attribute sets
@@ -349,6 +350,7 @@ class SpecFamily(SpecElement):
         super().__init__(self, spec)
 
         self.proto = self.yaml.get('protocol', 'genetlink')
+        self.msg_id_model = self.yaml['operations'].get('enum-model', 'unified')
 
         if schema_path is None:
             schema_path = os.path.dirname(os.path.dirname(spec_path)) + f'/{self.proto}.yaml'
@@ -442,6 +444,10 @@ class SpecFamily(SpecElement):
             else:
                 raise Exception("Can't parse directional ops")
 
+            if req_val == req_val_next:
+                req_val = None
+            if rsp_val == rsp_val_next:
+                rsp_val = None
             op = self.new_operation(elem, req_val, rsp_val)
             req_val = req_val_next
             rsp_val = rsp_val_next
@@ -473,10 +479,9 @@ class SpecFamily(SpecElement):
             attr_set = self.new_attr_set(elem)
             self.attr_sets[elem['name']] = attr_set
 
-        msg_id_model = self.yaml['operations'].get('enum-model', 'unified')
-        if msg_id_model == 'unified':
+        if self.msg_id_model == 'unified':
             self._dictify_ops_unified()
-        elif msg_id_model == 'directional':
+        elif self.msg_id_model == 'directional':
             self._dictify_ops_directional()
 
         for op in self.msgs.values():

tools/net/ynl/samples/.gitignore

+devlink
 netdev

tools/net/ynl/samples/devlink.c

+// SPDX-License-Identifier: GPL-2.0
+#include <stdio.h>
+#include <string.h>
+
+#include <ynl.h>
+
+#include "devlink-user.h"
+
+int main(int argc, char **argv)
+{
+	struct devlink_get_list *devs;
+	struct ynl_sock *ys;
+
+	ys = ynl_sock_create(&ynl_devlink_family, NULL);
+	if (!ys)
+		return 1;
+
+	devs = devlink_get_dump(ys);
+	if (!devs)
+		goto err_close;
+
+	ynl_dump_foreach(devs, d) {
+		struct devlink_info_get_req *info_req;
+		struct devlink_info_get_rsp *info_rsp;
+
+		printf("%s/%s:\n", d->bus_name, d->dev_name);
+
+		info_req = devlink_info_get_req_alloc();
+		devlink_info_get_req_set_bus_name(info_req, d->bus_name);
+		devlink_info_get_req_set_dev_name(info_req, d->dev_name);
+
+		info_rsp = devlink_info_get(ys, info_req);
+		devlink_info_get_req_free(info_req);
+		if (!info_rsp)
+			goto err_free_devs;
+
+		if (info_rsp->_present.info_driver_name_len)
+			printf("    driver: %s\n", info_rsp->info_driver_name);
+		if (info_rsp->n_info_version_running)
+			printf("    running fw:\n");
+		for (unsigned i = 0; i < info_rsp->n_info_version_running; i++)
+			printf("        %s: %s\n",
+			       info_rsp->info_version_running[i].info_version_name,
+			       info_rsp->info_version_running[i].info_version_value);
+		printf("    ...\n");
+		devlink_info_get_rsp_free(info_rsp);
+	}
+	devlink_get_list_free(devs);
+
+	ynl_sock_destroy(ys);
+
+	return 0;
+
+err_free_devs:
+	devlink_get_list_free(devs);
+err_close:
+	fprintf(stderr, "YNL: %s\n", ys->err.msg);
+	ynl_sock_destroy(ys);
+	return 2;
+}

tools/net/ynl/ynl-gen-c.py

@@ -709,9 +709,6 @@ class Operation(SpecOperation):
     def __init__(self, family, yaml, req_value, rsp_value):
         super().__init__(family, yaml, req_value, rsp_value)
 
-        if req_value != rsp_value:
-            raise Exception("Directional messages not supported by codegen")
-
         self.render_name = family.name + '_' + c_lower(self.name)
 
         self.dual_policy = ('do' in yaml and 'request' in yaml['do']) and \
@@ -857,26 +854,73 @@ class Family(SpecFamily):
                 self.root_sets[op['attribute-set']]['reply'].update(rsp_attrs)
 
     def _load_nested_sets(self):
+        attr_set_queue = list(self.root_sets.keys())
+        attr_set_seen = set(self.root_sets.keys())
+
+        while len(attr_set_queue):
+            a_set = attr_set_queue.pop(0)
+            for attr, spec in self.attr_sets[a_set].items():
+                if 'nested-attributes' not in spec:
+                    continue
+
+                nested = spec['nested-attributes']
+                if nested not in attr_set_seen:
+                    attr_set_queue.append(nested)
+                    attr_set_seen.add(nested)
+
+                inherit = set()
+                if nested not in self.root_sets:
+                    if nested not in self.pure_nested_structs:
+                        self.pure_nested_structs[nested] = Struct(self, nested, inherited=inherit)
+                else:
+                    raise Exception(f'Using attr set as root and nested not supported - {nested}')
+
+                if 'type-value' in spec:
+                    if nested in self.root_sets:
+                        raise Exception("Inheriting members to a space used as root not supported")
+                    inherit.update(set(spec['type-value']))
+                elif spec['type'] == 'array-nest':
+                    inherit.add('idx')
+                self.pure_nested_structs[nested].set_inherited(inherit)
+
         for root_set, rs_members in self.root_sets.items():
             for attr, spec in self.attr_sets[root_set].items():
                 if 'nested-attributes' in spec:
-                    inherit = set()
                     nested = spec['nested-attributes']
-                    if nested not in self.root_sets:
-                        if nested not in self.pure_nested_structs:
-                            self.pure_nested_structs[nested] = Struct(self, nested, inherited=inherit)
                     if attr in rs_members['request']:
                         self.pure_nested_structs[nested].request = True
                     if attr in rs_members['reply']:
                         self.pure_nested_structs[nested].reply = True
 
-                    if 'type-value' in spec:
-                        if nested in self.root_sets:
-                            raise Exception("Inheriting members to a space used as root not supported")
-                        inherit.update(set(spec['type-value']))
-                    elif spec['type'] == 'array-nest':
-                        inherit.add('idx')
-                    self.pure_nested_structs[nested].set_inherited(inherit)
+        # Try to reorder according to dependencies
+        pns_key_list = list(self.pure_nested_structs.keys())
+        pns_key_seen = set()
+        rounds = len(pns_key_list)**2  # it's basically bubble sort
+        for _ in range(rounds):
+            if len(pns_key_list) == 0:
+                break
+            name = pns_key_list.pop(0)
+            finished = True
+            for _, spec in self.attr_sets[name].items():
+                if 'nested-attributes' in spec:
+                    if spec['nested-attributes'] not in pns_key_seen:
+                        # Dicts are sorted, this will make struct last
+                        struct = self.pure_nested_structs.pop(name)
+                        self.pure_nested_structs[name] = struct
+                        finished = False
+                        break
+            if finished:
+                pns_key_seen.add(name)
+            else:
+                pns_key_list.append(name)
+        # Propagate the request / reply
+        for attr_set, struct in reversed(self.pure_nested_structs.items()):
+            for _, spec in self.attr_sets[attr_set].items():
+                if 'nested-attributes' in spec:
+                    child = self.pure_nested_structs.get(spec['nested-attributes'])
+                    if child:
+                        child.request |= struct.request
+                        child.reply |= struct.reply
 
     def _load_all_notify(self):
         for op_name, op in self.ops.items():
@@ -1191,10 +1235,6 @@ def print_dump_prototype(ri):
     print_prototype(ri, "request")
 
 
-def put_typol_fwd(cw, struct):
-    cw.p(f'extern struct ynl_policy_nest {struct.render_name}_nest;')
-
-
 def put_typol(cw, struct):
     type_max = struct.attr_set.max_name
     cw.block_start(line=f'struct ynl_policy_attr {struct.render_name}_policy[{type_max} + 1] =')
@@ -1212,6 +1252,21 @@ def put_typol(cw, struct):
     cw.nl()
 
 
+def _put_enum_to_str_helper(cw, render_name, map_name, arg_name, enum=None):
+    args = [f'int {arg_name}']
+    if enum and not ('enum-name' in enum and not enum['enum-name']):
+        args = [f'enum {render_name} {arg_name}']
+    cw.write_func_prot('const char *', f'{render_name}_str', args)
+    cw.block_start()
+    if enum and enum.type == 'flags':
+        cw.p(f'{arg_name} = ffs({arg_name}) - 1;')
+    cw.p(f'if ({arg_name} < 0 || {arg_name} >= (int)MNL_ARRAY_SIZE({map_name}))')
+    cw.p('return NULL;')
+    cw.p(f'return {map_name}[{arg_name}];')
+    cw.block_end()
+    cw.nl()
+
+
 def put_op_name_fwd(family, cw):
     cw.write_func_prot('const char *', f'{family.name}_op_str', ['int op'], suffix=';')
 
@@ -1220,17 +1275,15 @@ def put_op_name(family, cw):
     map_name = f'{family.name}_op_strmap'
     cw.block_start(line=f"static const char * const {map_name}[] =")
     for op_name, op in family.msgs.items():
-        cw.p(f'[{op.enum_name}] = "{op_name}",')
+        if op.rsp_value:
+            if op.req_value == op.rsp_value:
+                cw.p(f'[{op.enum_name}] = "{op_name}",')
+            else:
+                cw.p(f'[{op.rsp_value}] = "{op_name}",')
     cw.block_end(line=';')
     cw.nl()
 
-    cw.write_func_prot('const char *', f'{family.name}_op_str', ['int op'])
-    cw.block_start()
-    cw.p(f'if (op < 0 || op >= (int)MNL_ARRAY_SIZE({map_name}))')
-    cw.p('return NULL;')
-    cw.p(f'return {map_name}[op];')
-    cw.block_end()
-    cw.nl()
+    _put_enum_to_str_helper(cw, family.name + '_op', map_name, 'op')
 
 
 def put_enum_to_str_fwd(family, cw, enum):
@@ -1248,18 +1301,7 @@ def put_enum_to_str(family, cw, enum):
     cw.block_end(line=';')
     cw.nl()
 
-    args = [f'enum {enum.render_name} value']
-    if 'enum-name' in enum and not enum['enum-name']:
-        args = ['int value']
-    cw.write_func_prot('const char *', f'{enum.render_name}_str', args)
-    cw.block_start()
-    if enum.type == 'flags':
-        cw.p('value = ffs(value) - 1;')
-    cw.p(f'if (value < 0 || value >= (int)MNL_ARRAY_SIZE({map_name}))')
-    cw.p('return NULL;')
-    cw.p(f'return {map_name}[value];')
-    cw.block_end()
-    cw.nl()
+    _put_enum_to_str_helper(cw, enum.render_name, map_name, 'value', enum=enum)
 
 
 def put_req_nested(ri, struct):
@@ -2241,6 +2283,13 @@ def main():
         os.sys.exit(1)
         return
 
+    supported_models = ['unified']
+    if args.mode == 'user':
+        supported_models += ['directional']
+    if parsed.msg_id_model not in supported_models:
+        print(f'Message enum-model {parsed.msg_id_model} not supported for {args.mode} generation')
+        os.sys.exit(1)
+
     cw = CodeWriter(BaseNlLib(), out_file)
 
     _, spec_kernel = find_kernel_root(args.spec)
@@ -2373,7 +2422,7 @@ def main():
             cw.nl()
 
             cw.p('/* Common nested types */')
-            for attr_set, struct in sorted(parsed.pure_nested_structs.items()):
+            for attr_set, struct in parsed.pure_nested_structs.items():
                 ri = RenderInfo(cw, parsed, args.mode, "", "", "", attr_set)
                 print_type_full(ri, struct)
 
@@ -2432,17 +2481,15 @@ def main():
             cw.nl()
 
             cw.p('/* Policies */')
-            for name, _ in parsed.attr_sets.items():
+            for name in parsed.pure_nested_structs:
                 struct = Struct(parsed, name)
-                put_typol_fwd(cw, struct)
-            cw.nl()
-
-            for name, _ in parsed.attr_sets.items():
+                put_typol(cw, struct)
+            for name in parsed.root_sets:
                 struct = Struct(parsed, name)
                 put_typol(cw, struct)
 
             cw.p('/* Common nested types */')
-            for attr_set, struct in sorted(parsed.pure_nested_structs.items()):
+            for attr_set, struct in parsed.pure_nested_structs.items():
                 ri = RenderInfo(cw, parsed, args.mode, "", "", "", attr_set)
 
                 free_rsp_nested(ri, struct)